Hugging Face's logo

comaniac
/
Mixtral-8x22B-Instruct-v0.1-FP8-v2

Text Generation
Transformers
Safetensors
mixtral
conversational
text-generation-inference
Inference Endpoints
fp8
Model card Files Community
comaniac commited on
Commit
fee9f43
•
1 Parent(s): b1fde83

Upload folder using huggingface_hub

Browse files
Files changed (36) hide show
  1. README.md +11 -0
  2. config.json +94 -0
  3. generation_config.json +6 -0
  4. model-00001-of-00029.safetensors +3 -0
  5. model-00002-of-00029.safetensors +3 -0
  6. model-00003-of-00029.safetensors +3 -0
  7. model-00004-of-00029.safetensors +3 -0
  8. model-00005-of-00029.safetensors +3 -0
  9. model-00006-of-00029.safetensors +3 -0
  10. model-00007-of-00029.safetensors +3 -0
  11. model-00008-of-00029.safetensors +3 -0
  12. model-00009-of-00029.safetensors +3 -0
  13. model-00010-of-00029.safetensors +3 -0
  14. model-00011-of-00029.safetensors +3 -0
  15. model-00012-of-00029.safetensors +3 -0
  16. model-00013-of-00029.safetensors +3 -0
  17. model-00014-of-00029.safetensors +3 -0
  18. model-00015-of-00029.safetensors +3 -0
  19. model-00016-of-00029.safetensors +3 -0
  20. model-00017-of-00029.safetensors +3 -0
  21. model-00018-of-00029.safetensors +3 -0
  22. model-00019-of-00029.safetensors +3 -0
  23. model-00020-of-00029.safetensors +3 -0
  24. model-00021-of-00029.safetensors +3 -0
  25. model-00022-of-00029.safetensors +3 -0
  26. model-00023-of-00029.safetensors +3 -0
  27. model-00024-of-00029.safetensors +3 -0
  28. model-00025-of-00029.safetensors +3 -0
  29. model-00026-of-00029.safetensors +3 -0
  30. model-00027-of-00029.safetensors +3 -0
  31. model-00028-of-00029.safetensors +3 -0
  32. model-00029-of-00029.safetensors +3 -0
  33. model.safetensors.index.json +0 -0
  34. special_tokens_map.json +23 -0
  35. tokenizer.json +0 -0
  36. tokenizer_config.json +108 -0
README.md ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## Mixtral-8x22B-Instruct-v0.1-FP8-v1
2
+
3
+ * Weights and activations are per-tensor quantized to float8_e4m3.
4
+ * Quantization with AutoFP8.
5
+ * Calibration dataset: Ultrachat (mgoin/ultrachat_2k)
6
+ * Samples: 2048
7
+ * Sequence length: 8192
8
+
9
+ ## Evaluation
10
+
11
+ TBA
config.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "mistralai/Mixtral-8x22B-Instruct-v0.1",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 6144,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 16384,
13
+ "max_position_embeddings": 65536,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 48,
16
+ "num_experts_per_tok": 2,
17
+ "num_hidden_layers": 56,
18
+ "num_key_value_heads": 8,
19
+ "num_local_experts": 8,
20
+ "output_router_logits": false,
21
+ "quantization_config": {
22
+ "activation_scheme": "static",
23
+ "ignored_layers": [
24
+ "model.layers.46.block_sparse_moe.gate",
25
+ "model.layers.54.block_sparse_moe.gate",
26
+ "model.layers.39.block_sparse_moe.gate",
27
+ "model.layers.8.block_sparse_moe.gate",
28
+ "model.layers.14.block_sparse_moe.gate",
29
+ "model.layers.12.block_sparse_moe.gate",
30
+ "model.layers.13.block_sparse_moe.gate",
31
+ "model.layers.5.block_sparse_moe.gate",
32
+ "model.layers.40.block_sparse_moe.gate",
33
+ "model.layers.31.block_sparse_moe.gate",
34
+ "model.layers.7.block_sparse_moe.gate",
35
+ "model.layers.28.block_sparse_moe.gate",
36
+ "model.layers.11.block_sparse_moe.gate",
37
+ "model.layers.25.block_sparse_moe.gate",
38
+ "model.layers.29.block_sparse_moe.gate",
39
+ "model.layers.37.block_sparse_moe.gate",
40
+ "model.layers.44.block_sparse_moe.gate",
41
+ "model.layers.50.block_sparse_moe.gate",
42
+ "model.layers.17.block_sparse_moe.gate",
43
+ "model.layers.26.block_sparse_moe.gate",
44
+ "model.layers.34.block_sparse_moe.gate",
45
+ "model.layers.42.block_sparse_moe.gate",
46
+ "model.layers.4.block_sparse_moe.gate",
47
+ "model.layers.36.block_sparse_moe.gate",
48
+ "model.layers.51.block_sparse_moe.gate",
49
+ "lm_head",
50
+ "model.layers.18.block_sparse_moe.gate",
51
+ "model.layers.43.block_sparse_moe.gate",
52
+ "model.layers.53.block_sparse_moe.gate",
53
+ "model.layers.35.block_sparse_moe.gate",
54
+ "model.layers.6.block_sparse_moe.gate",
55
+ "model.layers.15.block_sparse_moe.gate",
56
+ "model.layers.22.block_sparse_moe.gate",
57
+ "model.layers.0.block_sparse_moe.gate",
58
+ "model.layers.38.block_sparse_moe.gate",
59
+ "model.layers.48.block_sparse_moe.gate",
60
+ "model.layers.2.block_sparse_moe.gate",
61
+ "model.layers.41.block_sparse_moe.gate",
62
+ "model.layers.21.block_sparse_moe.gate",
63
+ "model.layers.20.block_sparse_moe.gate",
64
+ "model.layers.49.block_sparse_moe.gate",
65
+ "model.layers.10.block_sparse_moe.gate",
66
+ "model.layers.32.block_sparse_moe.gate",
67
+ "model.layers.33.block_sparse_moe.gate",
68
+ "model.layers.30.block_sparse_moe.gate",
69
+ "model.layers.19.block_sparse_moe.gate",
70
+ "model.layers.3.block_sparse_moe.gate",
71
+ "model.layers.55.block_sparse_moe.gate",
72
+ "model.layers.23.block_sparse_moe.gate",
73
+ "model.layers.9.block_sparse_moe.gate",
74
+ "model.layers.45.block_sparse_moe.gate",
75
+ "model.layers.47.block_sparse_moe.gate",
76
+ "model.layers.1.block_sparse_moe.gate",
77
+ "model.layers.27.block_sparse_moe.gate",
78
+ "model.layers.16.block_sparse_moe.gate",
79
+ "model.layers.52.block_sparse_moe.gate",
80
+ "model.layers.24.block_sparse_moe.gate"
81
+ ],
82
+ "quant_method": "fp8"
83
+ },
84
+ "rms_norm_eps": 1e-05,
85
+ "rope_theta": 1000000.0,
86
+ "router_aux_loss_coef": 0.001,
87
+ "router_jitter_noise": 0.0,
88
+ "sliding_window": null,
89
+ "tie_word_embeddings": false,
90
+ "torch_dtype": "bfloat16",
91
+ "transformers_version": "4.40.2",
92
+ "use_cache": true,
93
+ "vocab_size": 32768
94
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.40.2"
6
+ }
model-00001-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:257b2f89c11862b54effd9f0eca27b59ee9775d3d3d3f48390d58dae1e1db1b8
3
+ size 4907575664
model-00002-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:707fefbe620a7174bddca52b8775339b170a33e806e5ea00bc5986943cd85640
3
+ size 4907601776
model-00003-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc0f42e5585852a8224dca00e834fa6ff9b7be9145962cca1891c95debe08e64
3
+ size 4907601776
model-00004-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:730d33f5010326be63f9c5d55c5708ba070d76ef9bbd81c714f07212e1fab93d
3
+ size 4907601776
model-00005-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3020248e76c2ebd99cd87fb763a36e43b4c2d65c0643c3be7ca5bcc915cc6a6d
3
+ size 4907601776
model-00006-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e55a4a35652a7be8ec0e35dced5a4c8323bcc5d0a7ad3363508a5b6a135cf66e
3
+ size 4907601920
model-00007-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9db2d9e4971ec0ae6be79af9ac501096bc87824f466e188e7d836e90b67ba5df
3
+ size 4907601944
model-00008-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c2e0fee64f2b8e6687cf44b98108b3189414f9ea7934c931c6df6b8d17a43b8d
3
+ size 4907601944
model-00009-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f09a6f135aa421d7935632c9d31ca021a8a41f730896545a12b0b2c75deb7054
3
+ size 4907601944
model-00010-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:049f5862b63c2871974f804675d175701da26e9bcc3e76410c70c33c456cffd9
3
+ size 4907601944
model-00011-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f53f8fd02b05f9cf3a82c8ba5fa2468130715117794f6b0dc068b153a9ae3412
3
+ size 4907601944
model-00012-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f9858c5f554ca87addd9899e2f034eea53ab36c18bdd0ac25cdb7cc97782428
3
+ size 4907601944
model-00013-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77a09672d369a04e0c3a8687929fdc0ec46f10b082fecc8327ad95994e4efa7d
3
+ size 4907601944
model-00014-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:535c84f4ded00f25f1dffd7fa88f42d33a52d11c6018ed4c486699692fcd43ae
3
+ size 4907601944
model-00015-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d377b4c1e68f9123c616a2df9cb1aa2b1a70094b460f9e9eb59adc7dc9e739cf
3
+ size 4907601944
model-00016-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5323d80bf59a5b510ac0a9b003b6bc10942df337d159cff0794b2bf3d2e44c0
3
+ size 4907601944
model-00017-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2492b00d5e62b8876e8e0b331e679068f3c856abd87ce0905fc8875ffc15850
3
+ size 4907601944
model-00018-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6a636fb79128d72c18fcd3475c2c0702c7856c8271c019abadad8d2bfc78645
3
+ size 4907601944
model-00019-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f17737fce41c4a66bd4fcc892ecb26ca431dcab5dc737d2783d62a607d5f52d
3
+ size 4907601944
model-00020-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0adce34b59afbbd91e461c87fde44d027fbba789b162f71eb825079e52090708
3
+ size 4907601944
model-00021-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8255a3ccbba16bdcc3215794ff8632ffb13f64d63c1720e611271b51adba3f71
3
+ size 4970418128
model-00022-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29eca6b4761c6ac1c189762708f59494837c92b9f8fa399fbab820269061f6f8
3
+ size 4995682048
model-00023-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:23e1b62aa137be0f895342e4e75c3a68b9ee5d75174973088c888ef169f78f58
3
+ size 4970516552
model-00024-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f0875a4032f04824130a62d364b5ea99d9ae6a9deca46cfe0e8841518836d8e
3
+ size 4907577160
model-00025-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f81ecb7c5fb47804f57e24d6cb9158c1b943d17d9b52feaeb035a0907071ba5
3
+ size 4907601944
model-00026-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d974238ac3654e08f16308979bf346d190b5e817a17d82fb880cd9ea74ebb4b6
3
+ size 4907601944
model-00027-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c18ff62a52e5c25915727dd7cb8b75c7dfdd50976095b23382f0c8d0ea2737f
3
+ size 4907601944
model-00028-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b18768f39354b90e70f5017fbf1c3706caf2e5e0411a6a7d8a5845858c2081a
3
+ size 4907601944
model-00029-of-00029.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07a15ef9548a6a6c6172bfe0eb446e69689a27781d83c8929e35ddba0b126caf
3
+ size 3410141576
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": true,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "3": {
30
+ "content": "[INST]",
31
+ "lstrip": false,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "4": {
38
+ "content": "[/INST]",
39
+ "lstrip": false,
40
+ "normalized": true,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "5": {
46
+ "content": "[TOOL_CALLS]",
47
+ "lstrip": false,
48
+ "normalized": true,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "6": {
54
+ "content": "[AVAILABLE_TOOLS]",
55
+ "lstrip": false,
56
+ "normalized": true,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "7": {
62
+ "content": "[/AVAILABLE_TOOLS]",
63
+ "lstrip": false,
64
+ "normalized": true,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "8": {
70
+ "content": "[TOOL_RESULTS]",
71
+ "lstrip": false,
72
+ "normalized": true,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "9": {
78
+ "content": "[/TOOL_RESULTS]",
79
+ "lstrip": false,
80
+ "normalized": true,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ }
85
+ },
86
+ "additional_special_tokens": [],
87
+ "bos_token": "<s>",
88
+ "chat_template": [
89
+ {
90
+ "name": "default",
91
+ "template": "{{bos_token}}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"
92
+ },
93
+ {
94
+ "name": "tool_use",
95
+ "template": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{% if tools %}{{'[AVAILABLE_TOOLS]'+ tools|string + '[/AVAILABLE_TOOLS]'}}{% endif %}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]'}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token}}{% endif %}{% endfor %}"
96
+ }
97
+ ],
98
+ "clean_up_tokenization_spaces": false,
99
+ "eos_token": "</s>",
100
+ "legacy": true,
101
+ "model_max_length": 1000000000000000019884624838656,
102
+ "pad_token": null,
103
+ "sp_model_kwargs": {},
104
+ "spaces_between_special_tokens": false,
105
+ "tokenizer_class": "LlamaTokenizer",
106
+ "unk_token": "<unk>",
107
+ "use_default_system_prompt": false
108
+ }