Initial commit

README.md

@@ -1,3 +1,22 @@
 ---
 license: apache-2.0
 ---
+# Mistral-7B-Instruct-v0.3 quantized to 4bits
+
+- weight-only quantization via GPTQ to 4bits
+- GPTQ optimized for X% accuracy recovery relative to the unquantized model
+
+# Open LLM Leaderboard evaluation scores
+
+|                      | Mistral-7B-Instruct-v0.3 | Mistral-7B-Instruct-v0.3-GPTQ-4bit<br>(this model) |
+| :------------------: | :----------------------: | :------------------------------------------------: |
+| arc-c<br>25-shot     | 63.48                    | 63.40                                              |
+| mmlu<br>5-shot       | 61.13                    | 60.89                                              |
+| hellaswag<br>10-shot | 84.49                    | ? gpu0                                             |
+| winogrande<br>5-shot | 79.16                    | 79.08                                              |
+| gsm8k<br>5-shot      | 43.37                    | 45.41                                              |
+| truthfulqa<br>0-shot | 59.65                    | 57.48                                              |
+| -------------------- | ------------------------ | -------------------------------------------------- |
+| Average<br>Accuracy  | 65.21                    |              x                                     |
+| Recovery             | 100%                     |              x                                     |
+

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "quantization_config": {
+    "bits": 4,
+    "damp_percent": 0.01,
+    "desc_act": false,
+    "group_size": 128,
+    "is_marlin_format": false,
+    "model_file_base_name": "model",
+    "model_name_or_path": null,
+    "quant_method": "gptq",
+    "static_groups": false,
+    "sym": true,
+    "true_sequential": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "vocab_size": 32768
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:547d383d403d546507b7a7f46692e3229f6ba2379f349f47f73a9f92754afdfc
+size 4171245160

quantize_config.json

@@ -0,0 +1,13 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "damp_percent": 0.01,
+  "desc_act": false,
+  "static_groups": false,
+  "sym": true,
+  "true_sequential": true,
+  "model_name_or_path": null,
+  "model_file_base_name": "model",
+  "is_marlin_format": false,
+  "quant_method": "gptq"
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404