Upload MistralForCausalLM

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "unsloth/mistral-7b-instruct-v0.2-bnb-4bit",
+  "_name_or_path": "quant-awq",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -15,26 +15,24 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "pad_token_id": 0,
   "quantization_config": {
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
+    "backend": "autoawq",
+    "bits": 4,
+    "do_fuse": false,
+    "fuse_max_seq_len": null,
+    "group_size": 128,
+    "modules_to_fuse": null,
+    "modules_to_not_convert": null,
+    "quant_method": "awq",
+    "version": "gemm",
+    "zero_point": true
   },
   "rms_norm_eps": 1e-05,
   "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.38.2",
-  "unsloth_version": "2024.4",
   "use_cache": true,
   "vocab_size": 32000
 }

generation_config.json

@@ -1,6 +1,7 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
+  "do_sample": true,
   "eos_token_id": 2,
   "transformers_version": "4.38.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:560e60c799fa7aeb3e45f5df038d7b8df60a25cf59aeaf0012f80d6a5eacc4f5
-size 4125687693
+oid sha256:dd6722c35d4396fc352363e28f9bb9ad05475aa1d72f1134a23e3c924d6d7ebf
+size 4150880232