Hello

README.md

@@ -1,15 +1,120 @@
-EXL2 quants of [Mixtral 8x22B Instruct v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1/tree/main)
+---
+license: apache-2.0
+---
 
-More bitrates still cooking
+# Model Card for Mixtral-8x22B-Instruct-v0.1
+The Mixtral-8x22B-Instruct-v0.1 Large Language Model (LLM) is an instruct fine-tuned version of the [Mixtral-8x22B-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-v0.1).
 
-[2.30 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/2.3bpw)     
-[2.50 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/2.5bpw)     
-[2.70 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/2.7bpw)     
-[3.00 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/3.0bpw)     
-[3.50 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/3.5bpw)     
-[4.00 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/4.0bpw)     
-[4.50 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/4.5bpw)     
-[5.00 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/5.0bpw)     
-[6.00 bits per weight](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/tree/6.0bpw)     
+## Run the model 
+```python
+from transformers import AutoModelForCausalLM
+from mistral_common.protocol.instruct.messages import (
+    AssistantMessage,
+    UserMessage,
+)
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+from mistral_common.tokens.instruct.normalize import ChatCompletionRequest
 
-[measurement.json](https://huggingface.co/turboderp/Mixtral-8x22B-Instruct-v0.1-exl2/blob/main/measurement.json)
+device = "cuda" # the device to load the model onto
+
+tokenizer_v3 = MistralTokenizer.v3()
+
+mistral_query = ChatCompletionRequest(
+    tools=[
+        Tool(
+            function=Function(
+                name="get_current_weather",
+                description="Get the current weather",
+                parameters={
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "The city and state, e.g. San Francisco, CA",
+                        },
+                        "format": {
+                            "type": "string",
+                            "enum": ["celsius", "fahrenheit"],
+                            "description": "The temperature unit to use. Infer this from the users location.",
+                        },
+                    },
+                    "required": ["location", "format"],
+                },
+            )
+        )
+    ],
+    messages=[
+        UserMessage(content="What's the weather like today in Paris"),
+    ],
+    model="test",
+)
+
+encodeds = tokenizer_v3.encode_chat_completion(mistral_query).tokens
+model = AutoModelForCausalLM.from_pretrained("mistralai/Mixtral-8x22B-Instruct-v0.1")
+model_inputs = encodeds.to(device)
+model.to(device)
+
+generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+sp_tokenizer = tokenizer_v3.instruct_tokenizer.tokenizer
+decoded = sp_tokenizer.decode(generated_ids[0])
+print(decoded)
+```
+
+# Instruct tokenizer
+The HuggingFace tokenizer included in this release should match our own. To compare: 
+`pip install mistral-common`
+
+```py
+from mistral_common.protocol.instruct.messages import (
+    AssistantMessage,
+    UserMessage,
+)
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+from mistral_common.tokens.instruct.normalize import ChatCompletionRequest
+
+from transformers import AutoTokenizer
+
+tokenizer_v3 = MistralTokenizer.v3()
+
+mistral_query = ChatCompletionRequest(
+    messages=[
+        UserMessage(content="How many experts ?"),
+        AssistantMessage(content="8"),
+        UserMessage(content="How big ?"),
+        AssistantMessage(content="22B"),
+        UserMessage(content="Noice 🎉 !"),
+    ],
+    model="test",
+)
+hf_messages = mistral_query.model_dump()['messages']
+
+tokenized_mistral = tokenizer_v3.encode_chat_completion(mistral_query).tokens
+
+tokenizer_hf = AutoTokenizer.from_pretrained('mistralai/Mixtral-8x22B-Instruct-v0.1')
+tokenized_hf = tokenizer_hf.apply_chat_template(hf_messages, tokenize=True)
+
+assert tokenized_hf == tokenized_mistral
+```
+
+# Function calling and special tokens
+This tokenizer includes more special tokens, related to function calling : 
+- [TOOL_CALLS]
+- [AVAILABLE_TOOLS]
+- [/AVAILABLE_TOOLS]
+- [TOOL_RESULT]
+- [/TOOL_RESULTS]
+
+If you want to use this model with function calling, please be sure to apply it similarly to what is done in our [SentencePieceTokenizerV3](github.com/mistralai/mistral-common/...).
+
+# The Mistral AI Team
+Albert Jiang, Alexandre Sablayrolles, Alexis Tacnet, Antoine Roux,
+Arthur Mensch, Audrey Herblin-Stoop, Baptiste Bout, Baudouin de Monicault,
+Blanche Savary, Bam4d, Caroline Feldman, Devendra Singh Chaplot,
+Diego de las Casas, Eleonore Arcelin, Emma Bou Hanna, Etienne Metzger,
+Gianna Lengyel, Guillaume Bour, Guillaume Lample, Harizo Rajaona,
+Jean-Malo Delignon, Jia Li, Justus Murke, Louis Martin, Louis Ternon,
+Lucile Saulnier, Lélio Renard Lavaud, Margaret Jennings, Marie Pellat,
+Marie Torelli, Marie-Anne Lachaux, Nicolas Schuhl, Patrick von Platen,
+Pierre Stock, Sandeep Subramanian, Sophia Yang, Szymon Antoniak, Teven Le Scao,
+Thibaut Lavril, Timothée Lacroix, Théophile Gervet, Thomas Wang,
+Valera Nemychnikova, William El Sayed, William Marshall

config.json

@@ -0,0 +1,40 @@
+{
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000.0,
+    "router_aux_loss_coef": 0.001,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.38.0",
+    "use_cache": true,
+    "vocab_size": 32768,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 3.75,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.34.0.dev0"
+}

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

output-00001-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c2ff2ba5476b3df89330d08098251db24985d08020a613687664e20d9e792d
+size 8581924752

output-00002-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:161ec218c6a4e079570d9d0de990119f47d26860d816881e345185e09dda4365
+size 8573208432

output-00003-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a256f8d323069917b9fbca66fb5ce6c3b414e8f05dc3ed4c45286e1af524c50
+size 8558958648

output-00004-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c539862538fde337435a29f2954043b4bfe466b0453443543451f8adfa732fc
+size 8553708680

output-00005-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18aeef37fc63451e856d24eb808801d4371361e3383ca769fa845f63bd93f252
+size 8589855920

output-00006-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd50ac28545134deec452ec84e3234966380f12884c1108de902f1d6c79e1878
+size 8573248536

output-00007-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74b999904f011d77e9097437a4e0eec2d2a1569fa8c6c3224dc06d1771f2d52c
+size 8539662824

output-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:978d6eb3161eb053efecd9b4eff2dbec811239bca5ebbd81f3f8588d78b3d8a7
+size 6296987240

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "b_inst": "[INST]",
+  "e_inst": "[/INST]"
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{bos_token}}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}