Upload folder using huggingface_hub

README.md

@@ -0,0 +1,114 @@
+---
+license: apache-2.0
+tags:
+- moe
+- mergekit
+- merge
+- chinese
+- arabic
+- english
+- multilingual
+- german
+- french
+- openchat/openchat-3.5-1210
+- beowolx/CodeNinja-1.0-OpenChat-7B
+- maywell/PiVoT-0.1-Starling-LM-RP
+- WizardLM/WizardMath-7B-V1.1
+- davidkim205/komt-mistral-7b-v1
+- OpenBuddy/openbuddy-zephyr-7b-v14.1
+- manishiitg/open-aditi-hi-v1
+- VAGOsolutions/SauerkrautLM-7b-v1-mistral
+---
+
+# MetaModel_moex8
+
+This model is a Mixure of Experts (MoE) made with [mergekit](https://github.com/cg123/mergekit) (mixtral branch). It uses the following base models:
+* [openchat/openchat-3.5-1210](https://huggingface.co/openchat/openchat-3.5-1210)
+* [beowolx/CodeNinja-1.0-OpenChat-7B](https://huggingface.co/beowolx/CodeNinja-1.0-OpenChat-7B)
+* [maywell/PiVoT-0.1-Starling-LM-RP](https://huggingface.co/maywell/PiVoT-0.1-Starling-LM-RP)
+* [WizardLM/WizardMath-7B-V1.1](https://huggingface.co/WizardLM/WizardMath-7B-V1.1)
+* [davidkim205/komt-mistral-7b-v1](https://huggingface.co/davidkim205/komt-mistral-7b-v1)
+* [OpenBuddy/openbuddy-zephyr-7b-v14.1](https://huggingface.co/OpenBuddy/openbuddy-zephyr-7b-v14.1)
+* [manishiitg/open-aditi-hi-v1](https://huggingface.co/manishiitg/open-aditi-hi-v1)
+* [VAGOsolutions/SauerkrautLM-7b-v1-mistral](https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-v1-mistral)
+
+## 🧩 Configuration
+
+```yamlbase_model: mlabonne/NeuralMarcoro14-7B
+dtype: bfloat16
+experts:
+- positive_prompts:
+  - chat
+  - assistant
+  - tell me
+  - explain
+  source_model: openchat/openchat-3.5-1210
+- positive_prompts:
+  - code
+  - python
+  - javascript
+  - programming
+  - algorithm
+  source_model: beowolx/CodeNinja-1.0-OpenChat-7B
+- positive_prompts:
+  - storywriting
+  - write
+  - scene
+  - story
+  - character
+  source_model: maywell/PiVoT-0.1-Starling-LM-RP
+- positive_prompts:
+  - reason
+  - math
+  - mathematics
+  - solve
+  - count
+  source_model: WizardLM/WizardMath-7B-V1.1
+- positive_prompts:
+  - korean
+  - answer in korean
+  - korea
+  source_model: davidkim205/komt-mistral-7b-v1
+- positive_prompts:
+  - chinese
+  - china
+  - answer in chinese
+  source_model: OpenBuddy/openbuddy-zephyr-7b-v14.1
+- positive_prompts:
+  - hindi
+  - india
+  - hindu
+  - answer in hindi
+  source_model: manishiitg/open-aditi-hi-v1
+- positive_prompts:
+  - german
+  - germany
+  - answer in german
+  - deutsch
+  source_model: VAGOsolutions/SauerkrautLM-7b-v1-mistral
+gate_mode: hidden
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers bitsandbytes accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "gagan3012/MetaModel_moex8"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "jeonsworld/CarbonVillain-en-10.7B-v4",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4096,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,20 @@
+base_model: jeonsworld/CarbonVillain-en-10.7B-v4
+gate_mode: hidden
+dtype: bfloat16
+experts:
+  - source_model: gagan3012/MetaModel
+    positive_prompts: [""]
+  - source_model: jeonsworld/CarbonVillain-en-10.7B-v2
+    positive_prompts: [""]
+  - source_model: jeonsworld/CarbonVillain-en-10.7B-v4
+    positive_prompts: [""]
+  - source_model: TomGrc/FusionNet_linear
+    positive_prompts: [""]
+  - source_model: DopeorNope/SOLARC-M-10.7B
+    positive_prompts: [""]
+  - source_model: VAGOsolutions/SauerkrautLM-SOLAR-Instruct
+    positive_prompts: [""]
+  - source_model: upstage/SOLAR-10.7B-Instruct-v1.0
+    positive_prompts: [""]
+  - source_model: fblgit/UNA-SOLAR-10.7B-Instruct-v1.0
+    positive_prompts: [""]

model-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c291c150295140eec7ea7311fd37e1f9088157433927dff3d27b27d0bf1d6c68
+size 9919946528

model-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ba2d64b77ff9b8caa67ddefb1bfc1538c1814d7397bc7830d9158938ad4f8d
+size 9982454760

model-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e9943fb78210d4c454b6296d72bd3f9dbf36f941c5ab18e134e84e2d8712633
+size 9982454760

model-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1741eaf25cadd8012e7c78e296bb4867978dc705a6eadab24af59be5c42e44
+size 9982454760

model-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f659b411e07fb84ab64d3102c4e9489b385a41a5a2326aa4f63351d34463b2
+size 9982454720

model-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05c4494e41f97a5631b7c40466c8a1d5f189ede8849e64fdb25b5274673f89b3
+size 9982454712

model-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20cf440adba8567bb7aaa20994cc3f14dbbb43175847c4ae8d266a4a12a6122d
+size 9982454760

model-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9510068d4a0f9613de42e332434e4bc95c0a29ea235afc25fae2654445dfa8
+size 9982454760

model-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4621d9262f377ab5e6b239d38694c164db4260050d91e39f8919b2d654aaab2a
+size 9982454760

model-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5224818062e82caba6e78d9271c7e40f77aa5443ec89d9659a8127c97e4d873e
+size 9982454680

model-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c216266f0ea59c30890fc446be202c843b47f27660675124e73b592d3b0f5ce8
+size 9982454752

model-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2637cfb2b443386a1f16c7f4d107767c8e71ae329b040e35e36295f83c88b444
+size 9982454760

model-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53fe85f2c5bdb118e38b21a575ca8a96a9fd982a352f8546ed52179e044a1c99
+size 9982454760

model-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970799f420ba13bfbec0f360663d00895209f715f4930dd2ba7c07b0d9ef0aff
+size 9999648648

model-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040d97482c0940b1f31294c6639b5a4ccdb29a18fc04bc275e0e1844b71fe632
+size 137369408

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n'  + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}