BigHuggyD commited on
Commit
5f48231
1 Parent(s): 6ec5baa

Upload model weights (#1)

Browse files

- edb42224bc572b49f1e8992b0b30c26c5de1076822df597705fc5546a76196fd (bca5ed46c4f7bd1fd2e9706c128a5977f19f8c34)
- e6f2348b46f7d34edb06cf57b0b2c7fd4b2be0705d79c097ec72c4ba3c596593 (1f91df435e783f70e1e561b54b8d323c592b17e3)
- 392213417a588772ac7a2c359de89079f24fc017e0c0a119d3e6fdd29abf081c (581eea8340f8eac17962a96a519cf23f4b26a0b1)
- 26a84e0bf86806f7b110ce679e9cce06264acb683b2444c0090bf8064423b75f (e21bea75b64c511443f66b7f8a2677ed76c95711)
- 35fef131c896f8bb007b0a906e55c1f0992437207fa2796501fa55da9d690754 (e7206beaa99a0dfe58aef0dd0d9fed70e967be2a)
- 0601855fefbb5ff6790aded491d7f493878f9fb1604c4d7e7e49b5c96e15fe65 (3ec0116b7dbb95dabe1475f5fe13189fe86b2e62)
- 38802ec046d5869726d8ac841bcd16f22ab5f508c446ef04633905f2c89814eb (889751a1889a59bedd686f46517c0bc97cd09741)
- e3dc6f23a98062128bd24494c7a0273b6e7f43ad611a385e70885d3ca8c4ee0d (d288e88bc2ee8530208b35d7f99ab8d477b87f6c)
- ba02d62be15ed23bd07a1f59f26d9db3ca5e34a47a6184d22d6ec132f436e728 (b6995d5b2fd81a015fb91bd32fe50cac75faa689)
- 34543f058299f8bfdd69557ddd93848bd527f894a06697ff8c3c0bde2b5502b2 (c7013375a05a82fd7ef420a8d85cf340f2a9cc9c)
- 04e814efbd1be347c23502a336cf89ff6d75987960fb3abf33fcb294e1fb9b11 (c6f5644436c710cafe3135615bef993bc7164995)
- 11cd76c29f6fcf010530ecd04dfe335e4781452379d542b770d79069f5cea82f (c18835ea807547c2151577f0bccb0c8d07e72ba6)
- 063a2f575b800366f16e3595bef2e19a835c221c83fc4968a88aa7a8816fb334 (f33a0fea6e077ea7e99579dee076610ac365d841)
- 8f4c067aab3492b285656062b6d4b8852849747862bfde75135abf0fbd29cabe (42b63f2938a7994dc1a65213c01a0ccd2afbd6fe)
- 12fdf567fcf4da7d050c6323c966562ef20d5f775cdc569d266dcf750148cd02 (30f73ef7a7bfc003810df592099757446b4cf4fb)
- 46b10809b7ca4d15884c222da9c777078c3e10c8148e8e7bff2d097e6aa4be5e (620af7051489fe305ad722fade5bc060ed212627)

README.md ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model:
3
+ - alpindale/WizardLM-2-8x22B
4
+ - lightblue/Karasu-Mixtral-8x22B-v0.1
5
+ - openbmb/Eurux-8x22b-nca
6
+ - cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
7
+ - migtissera/Tess-2.0-Mixtral-8x22B
8
+ - fireworks-ai/mixtral-8x22b-instruct-oh
9
+ library_name: transformers
10
+ tags:
11
+ - mergekit
12
+ - merge
13
+
14
+ ---
15
+ # Untitled Model (1)
16
+
17
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
18
+
19
+ ## Merge Details
20
+ ### Merge Method
21
+
22
+ This model was merged using the [Model Stock](https://arxiv.org/abs/2403.19522) merge method using [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B) as a base.
23
+
24
+ ### Models Merged
25
+
26
+ The following models were included in the merge:
27
+ * [lightblue/Karasu-Mixtral-8x22B-v0.1](https://huggingface.co/lightblue/Karasu-Mixtral-8x22B-v0.1)
28
+ * [openbmb/Eurux-8x22b-nca](https://huggingface.co/openbmb/Eurux-8x22b-nca)
29
+ * [cognitivecomputations/dolphin-2.9.2-mixtral-8x22b](https://huggingface.co/cognitivecomputations/dolphin-2.9.2-mixtral-8x22b)
30
+ * [migtissera/Tess-2.0-Mixtral-8x22B](https://huggingface.co/migtissera/Tess-2.0-Mixtral-8x22B)
31
+ * [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
32
+
33
+ ### Configuration
34
+
35
+ The following YAML configuration was used to produce this model:
36
+
37
+ ```yaml
38
+ models:
39
+ - model: alpindale/WizardLM-2-8x22B
40
+ - model: lightblue/Karasu-Mixtral-8x22B-v0.1
41
+ - model: fireworks-ai/mixtral-8x22b-instruct-oh
42
+ - model: cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
43
+ - model: openbmb/Eurux-8x22b-nca
44
+ - model: migtissera/Tess-2.0-Mixtral-8x22B
45
+ base_model: alpindale/WizardLM-2-8x22B
46
+ merge_method: model_stock
47
+ dtype: bfloat16
48
+
49
+ ```
config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "alpindale/WizardLM-2-8x22B",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 6144,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 16384,
13
+ "max_position_embeddings": 65536,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 48,
16
+ "num_experts_per_tok": 2,
17
+ "num_hidden_layers": 56,
18
+ "num_key_value_heads": 8,
19
+ "num_local_experts": 8,
20
+ "output_router_logits": false,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_theta": 1000000,
23
+ "router_aux_loss_coef": 0.001,
24
+ "router_jitter_noise": 0.0,
25
+ "sliding_window": null,
26
+ "tie_word_embeddings": false,
27
+ "torch_dtype": "bfloat16",
28
+ "transformers_version": "4.41.2",
29
+ "use_cache": false,
30
+ "vocab_size": 32000,
31
+ "quantization_config": {
32
+ "quant_method": "exl2",
33
+ "version": "0.1.5",
34
+ "bits": 7.0,
35
+ "head_bits": 8,
36
+ "calibration": {
37
+ "rows": 100,
38
+ "length": 2048,
39
+ "dataset": "(default)"
40
+ }
41
+ }
42
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ models:
2
+ - model: alpindale/WizardLM-2-8x22B
3
+ - model: lightblue/Karasu-Mixtral-8x22B-v0.1
4
+ - model: fireworks-ai/mixtral-8x22b-instruct-oh
5
+ - model: cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
6
+ - model: openbmb/Eurux-8x22b-nca
7
+ - model: migtissera/Tess-2.0-Mixtral-8x22B
8
+ base_model: alpindale/WizardLM-2-8x22B
9
+ merge_method: model_stock
10
+ dtype: bfloat16
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
output-00001-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5a33d5b2c3ed369e53a73bc174c689fea5d3824ebefd8809b99c37760a39b67
3
+ size 8555032872
output-00002-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10ab4d94049730769bec2dd2d6a22e6b1e18888a5752dfd2b1109ff21045104b
3
+ size 8554100928
output-00003-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33093fa7b0517e4f59940888ea3eb7205609f08d3900ae3364d63ff03342ccd5
3
+ size 8552536336
output-00004-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b76c40e8d9226010db606107755b0620af89c43eb77b432feeea9e6610321a8
3
+ size 8519348008
output-00005-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2acf9d7b49ee50d351688a6166880331312776a2a665b9c11cb5a5d8217226b
3
+ size 8589913720
output-00006-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f32bf468914cbd2e2ea33876768d2d6c7dc3ab6cff9bd26669a8484b306b43d9
3
+ size 8547138648
output-00007-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea9478d81c69a27e417f03d67af8578153246f659ff37a3604708a41890c9d01
3
+ size 8538200136
output-00008-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f867aa54f0100b115f54895d62bc2289bdb4b93ec3667ccb40e90a65846e47f
3
+ size 8542235176
output-00009-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3a2cc8e0b128c822ece47e9ef9f9f2a08d8749bf3226636a3b9811b01b4dc17
3
+ size 8532070992
output-00010-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a94ba3f28509688bf78ce1fa56258d37abe2578f68f4bed21f263441679fa787
3
+ size 8586194816
output-00011-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e70b488ccebbba76987c05ae34a3415fbd622d8873d7482c776080e4444b3d24
3
+ size 8523347504
output-00012-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecd198e4edd069d3599f750bd852b0cbcf6519d9b3026ab23a66f466454c0cdd
3
+ size 8521339464
output-00013-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27c2ae8312b1028ff2b76151d1ec9e4f08a5db1140a0fa57b410e6d73765468e
3
+ size 8562404128
output-00014-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea4e7deaa35bc589e257b7dca5c90a8354df8b2a571b15f3b7c2575c86ebbc68
3
+ size 8587055456
output-00015-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8cd0307940a54df812b341ad9abdc8b97fb9f97b6f6ccb9ebea53527bca72202
3
+ size 2699322168
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "additional_special_tokens": [],
32
+ "bos_token": "<s>",
33
+ "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{{ messages[0]['content'].strip() }}{% else %}{% set loop_messages = messages %}{{ 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' }}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ ' USER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% else %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ '\nUSER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' ASSISTANT:' }}{% endif %}",
34
+ "clean_up_tokenization_spaces": false,
35
+ "eos_token": "</s>",
36
+ "legacy": true,
37
+ "model_max_length": 1000000000000000019884624838656,
38
+ "pad_token": "<unk>",
39
+ "padding_side": "right",
40
+ "sp_model_kwargs": {},
41
+ "spaces_between_special_tokens": false,
42
+ "tokenizer_class": "LlamaTokenizer",
43
+ "unk_token": "<unk>",
44
+ "use_default_system_prompt": true
45
+ }