Upload folder using huggingface_hub

README.md

@@ -1,3 +1,35 @@
----
-license: cc-by-nc-sa-4.0
----
+---
+base_model:
+- alpindale/WizardLM-2-8x22B
+- HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1
+library_name: transformers
+tags:
+- mergekit
+- merge
+license: cc-by-nc-sa-4.0
+---
+# merge
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Models Merged
+
+The following models were included in the merge:
+* [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B)
+* [HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1)
+
+## Benchmark results
+### 1. MT-Bench from lmsys
+We adapted the code from [FastChat](https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge) to benchmark our model with GPT-4 as a judge. Here is the result
+```markdown
+|        | Model                                   | Turn | Score    |
+|--------|-----------------------------------------|------|----------|
+| First  | tlphams/Wizard-Zephyr-Orpo-8x22B        | 1    | 9.1625   |
+|        | mistralai/Mixtral-8x22B-Instruct-v0.1   | 1    | 9.1500   |
+| Second | tlphams/Wizard-Zephyr-Orpo-8x22B        | 2    | 8.873418 |
+|        | mistralai/Mixtral-8x22B-Instruct-v0.1   | 2    | 8.250000 |
+| Average| tlphams/Wizard-Zephyr-Orpo-8x22B        |      | 9.018868 |
+|        | mistralai/Mixtral-8x22B-Instruct-v0.1   |      | 8.700000 |
+```
+The score is slightly lower than [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B), but still higher than GPT-4-0314. Then the research and experimental work still need to continue ^^

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "alpindale/WizardLM-2-8x22B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,8 @@
+models:
+  - model: alpindale/WizardLM-2-8x22B
+  - model: HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1
+merge_method: slerp
+base_model: alpindale/WizardLM-2-8x22B
+dtype: bfloat16
+parameters:
+  t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers

model-00001-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84597db45a1a6a8c79572566ecdfa282757a064bd5d361d450c824ee9d54f199
+size 135

model-00002-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:262b5c0b7c3da3fb99a217db08fc0a2cf0761bb6a6ed44dd18da0ef96a734eb6
+size 135

model-00003-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6687cc32c25bf8f9c6f57f09d8311979ef2dc092d77a65daf205639f16c087f6
+size 135

model-00004-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d86948170a07e47aba0c50fd8e56bc1abfcf09d4098c9a6afa8fc357b036d40
+size 135

model-00005-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a3f2c47ca7212e89259bb0f6f979fb31a826005a9bb7964c880423428aafe88
+size 135

model-00006-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be08696ae70a10ff329e39f696957377ffd69e7bb5b3dfdaf9c2b2d659fca63
+size 135

model-00007-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e65ea98b804911ed0958ea3e693ede36b4057017f1a144ad9079d6b2f97cff38
+size 135

model-00008-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48566bcb845b16077ba18b9366719b2230d4acb74bc7c8a03e985fc0a55da671
+size 135

model-00009-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1225f6ff0e71b166a25019069cb9b06c7e6b9911f1d986cc010c1599c699855
+size 135

model-00010-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59f0bce6626dd07bdbf3deff1cc61a02abf6f409e636ea5a10bf4a0d5c57bb9
+size 135

model-00011-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84672b7d7534c8edc4874be930bec76f5fe023fa9183ae1a99e049e850469945
+size 135

model-00012-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb44a942c36b0732e62649d9cd2d8491dd72a017b19952bcb128cadc0b60ffe5
+size 135

model-00013-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9baab75a925eea07bccb40681d39a633dfdbd7549935b97f49ac78b3c82974
+size 135

model-00014-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79869de99b90bf4454c1b5d9ee2357cdb6d02ddd1c8770c00049af83f7975db4
+size 135

model-00015-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc9ed55c98e21825232908394ca63f9ac4d4b2ab6e418ca53f668a1780bfad5c
+size 135

model-00016-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c89df436ed23ce769dd6f0203c5ae467f1a5b18d6bcd8077b0c833be89b59624
+size 135

model-00017-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac8472dbe771f0470ee367162121c6fe5e15e2b39e9da92f54955ae5243b0b52
+size 135

model-00018-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7496edfc88419c57da224a0104630f61a35511cc12f4852588771de810ad0f58
+size 135

model-00019-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:358f26f55f41621ff1657657075347293aa9ec6bb6c60d3500d1d7cd31c009a7
+size 135

model-00020-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad63dc3b64a5f1dcf7f00df74db3307664a4bde6b4eccd3f52be04f041bd829
+size 135

model-00021-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b0bf2526210b6160ef6ed3f33c42d7292629bdd0d2639c78efe4c0f09119c75
+size 135

model-00022-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6babc7b854a101e97edbe7a22b4001fb7a9f89e669987e14a211f1224a1c22a
+size 135

model-00023-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b0888194d002c1982977eb6a78bfa4f4ed5c0356b58c8b5f4ce85830fcf6fc3
+size 135

model-00024-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e04adbe644b0f1b4cde5ebcd362dacc9f45fe9f3a4c565bc569180c6089508
+size 135

model-00025-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18e1b9a1660c314aedf8c9d598cd799395611022caba078bb96714e41f9cbfb7
+size 135

model-00026-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4679dbb64984cf4663a71941ffbaed935a9e6c6b80e19282fc21fba2f6f058b9
+size 135

model-00027-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5709126e94775d232a46a1147b5ca3d7fe388e6befa3e88c6be391132a51c92
+size 135

model-00028-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00b5b5241a9b219e7b8e8f8e4d609d02079c41186ab8704671376ed488236a50
+size 135

model-00029-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6367708d40066d89684447d77c8deeaccbde3f8b79766568869f5b3ea256a8b4
+size 135

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3daefa6fd9ee26430a71ad6009f05c4c4ec086746b2dcc3d04649f631d3654f
+size 131

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{{ messages[0]['content'].strip() }}{% else %}{% set loop_messages = messages %}{{ 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' }}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ ' USER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% else %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ '\nUSER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' ASSISTANT:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}