Upload model weights (#1)

- edb42224bc572b49f1e8992b0b30c26c5de1076822df597705fc5546a76196fd (bca5ed46c4f7bd1fd2e9706c128a5977f19f8c34)
- e6f2348b46f7d34edb06cf57b0b2c7fd4b2be0705d79c097ec72c4ba3c596593 (1f91df435e783f70e1e561b54b8d323c592b17e3)
- 392213417a588772ac7a2c359de89079f24fc017e0c0a119d3e6fdd29abf081c (581eea8340f8eac17962a96a519cf23f4b26a0b1)
- 26a84e0bf86806f7b110ce679e9cce06264acb683b2444c0090bf8064423b75f (e21bea75b64c511443f66b7f8a2677ed76c95711)
- 35fef131c896f8bb007b0a906e55c1f0992437207fa2796501fa55da9d690754 (e7206beaa99a0dfe58aef0dd0d9fed70e967be2a)
- 0601855fefbb5ff6790aded491d7f493878f9fb1604c4d7e7e49b5c96e15fe65 (3ec0116b7dbb95dabe1475f5fe13189fe86b2e62)
- 38802ec046d5869726d8ac841bcd16f22ab5f508c446ef04633905f2c89814eb (889751a1889a59bedd686f46517c0bc97cd09741)
- e3dc6f23a98062128bd24494c7a0273b6e7f43ad611a385e70885d3ca8c4ee0d (d288e88bc2ee8530208b35d7f99ab8d477b87f6c)
- ba02d62be15ed23bd07a1f59f26d9db3ca5e34a47a6184d22d6ec132f436e728 (b6995d5b2fd81a015fb91bd32fe50cac75faa689)
- 34543f058299f8bfdd69557ddd93848bd527f894a06697ff8c3c0bde2b5502b2 (c7013375a05a82fd7ef420a8d85cf340f2a9cc9c)
- 04e814efbd1be347c23502a336cf89ff6d75987960fb3abf33fcb294e1fb9b11 (c6f5644436c710cafe3135615bef993bc7164995)
- 11cd76c29f6fcf010530ecd04dfe335e4781452379d542b770d79069f5cea82f (c18835ea807547c2151577f0bccb0c8d07e72ba6)
- 063a2f575b800366f16e3595bef2e19a835c221c83fc4968a88aa7a8816fb334 (f33a0fea6e077ea7e99579dee076610ac365d841)
- 8f4c067aab3492b285656062b6d4b8852849747862bfde75135abf0fbd29cabe (42b63f2938a7994dc1a65213c01a0ccd2afbd6fe)
- 12fdf567fcf4da7d050c6323c966562ef20d5f775cdc569d266dcf750148cd02 (30f73ef7a7bfc003810df592099757446b4cf4fb)
- 46b10809b7ca4d15884c222da9c777078c3e10c8148e8e7bff2d097e6aa4be5e (620af7051489fe305ad722fade5bc060ed212627)

README.md

@@ -0,0 +1,49 @@
+---
+base_model:
+- alpindale/WizardLM-2-8x22B
+- lightblue/Karasu-Mixtral-8x22B-v0.1
+- openbmb/Eurux-8x22b-nca
+- cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
+- migtissera/Tess-2.0-Mixtral-8x22B
+- fireworks-ai/mixtral-8x22b-instruct-oh
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# Untitled Model (1)
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the [Model Stock](https://arxiv.org/abs/2403.19522) merge method using [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B) as a base.
+
+### Models Merged
+
+The following models were included in the merge:
+* [lightblue/Karasu-Mixtral-8x22B-v0.1](https://huggingface.co/lightblue/Karasu-Mixtral-8x22B-v0.1)
+* [openbmb/Eurux-8x22b-nca](https://huggingface.co/openbmb/Eurux-8x22b-nca)
+* [cognitivecomputations/dolphin-2.9.2-mixtral-8x22b](https://huggingface.co/cognitivecomputations/dolphin-2.9.2-mixtral-8x22b)
+* [migtissera/Tess-2.0-Mixtral-8x22B](https://huggingface.co/migtissera/Tess-2.0-Mixtral-8x22B)
+* [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+models:
+  - model: alpindale/WizardLM-2-8x22B
+  - model: lightblue/Karasu-Mixtral-8x22B-v0.1
+  - model: fireworks-ai/mixtral-8x22b-instruct-oh
+  - model: cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
+  - model: openbmb/Eurux-8x22b-nca
+  - model: migtissera/Tess-2.0-Mixtral-8x22B
+base_model: alpindale/WizardLM-2-8x22B
+merge_method: model_stock
+dtype: bfloat16
+
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "alpindale/WizardLM-2-8x22B",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.41.2",
+    "use_cache": false,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.1.5",
+        "bits": 7.0,
+        "head_bits": 8,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

mergekit_config.yml

@@ -0,0 +1,10 @@
+models:
+  - model: alpindale/WizardLM-2-8x22B
+  - model: lightblue/Karasu-Mixtral-8x22B-v0.1
+  - model: fireworks-ai/mixtral-8x22b-instruct-oh
+  - model: cognitivecomputations/dolphin-2.9.2-mixtral-8x22b
+  - model: openbmb/Eurux-8x22b-nca
+  - model: migtissera/Tess-2.0-Mixtral-8x22B
+base_model: alpindale/WizardLM-2-8x22B
+merge_method: model_stock
+dtype: bfloat16

output-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a33d5b2c3ed369e53a73bc174c689fea5d3824ebefd8809b99c37760a39b67
+size 8555032872

output-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10ab4d94049730769bec2dd2d6a22e6b1e18888a5752dfd2b1109ff21045104b
+size 8554100928

output-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33093fa7b0517e4f59940888ea3eb7205609f08d3900ae3364d63ff03342ccd5
+size 8552536336

output-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b76c40e8d9226010db606107755b0620af89c43eb77b432feeea9e6610321a8
+size 8519348008

output-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2acf9d7b49ee50d351688a6166880331312776a2a665b9c11cb5a5d8217226b
+size 8589913720

output-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f32bf468914cbd2e2ea33876768d2d6c7dc3ab6cff9bd26669a8484b306b43d9
+size 8547138648

output-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea9478d81c69a27e417f03d67af8578153246f659ff37a3604708a41890c9d01
+size 8538200136

output-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f867aa54f0100b115f54895d62bc2289bdb4b93ec3667ccb40e90a65846e47f
+size 8542235176

output-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3a2cc8e0b128c822ece47e9ef9f9f2a08d8749bf3226636a3b9811b01b4dc17
+size 8532070992

output-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94ba3f28509688bf78ce1fa56258d37abe2578f68f4bed21f263441679fa787
+size 8586194816

output-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e70b488ccebbba76987c05ae34a3415fbd622d8873d7482c776080e4444b3d24
+size 8523347504

output-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd198e4edd069d3599f750bd852b0cbcf6519d9b3026ab23a66f466454c0cdd
+size 8521339464

output-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27c2ae8312b1028ff2b76151d1ec9e4f08a5db1140a0fa57b410e6d73765468e
+size 8562404128

output-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea4e7deaa35bc589e257b7dca5c90a8354df8b2a571b15f3b7c2575c86ebbc68
+size 8587055456

output-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cd0307940a54df812b341ad9abdc8b97fb9f97b6f6ccb9ebea53527bca72202
+size 2699322168

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{{ messages[0]['content'].strip() }}{% else %}{% set loop_messages = messages %}{{ 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' }}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ ' USER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% else %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ '\nUSER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' ASSISTANT:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}