Upload folder using huggingface_hub

README.md

@@ -0,0 +1,114 @@
+---
+license: apache-2.0
+tags:
+- moe
+- mergekit
+- merge
+- chinese
+- arabic
+- english
+- multilingual
+- german
+- french
+- openchat/openchat-3.5-1210
+- beowolx/CodeNinja-1.0-OpenChat-7B
+- maywell/PiVoT-0.1-Starling-LM-RP
+- WizardLM/WizardMath-7B-V1.1
+- davidkim205/komt-mistral-7b-v1
+- OpenBuddy/openbuddy-zephyr-7b-v14.1
+- manishiitg/open-aditi-hi-v1
+- VAGOsolutions/SauerkrautLM-7b-v1-mistral
+---
+
+# MetaModel_moe_multilingualv2
+
+This model is a Mixure of Experts (MoE) made with [mergekit](https://github.com/cg123/mergekit) (mixtral branch). It uses the following base models:
+* [openchat/openchat-3.5-1210](https://huggingface.co/openchat/openchat-3.5-1210)
+* [beowolx/CodeNinja-1.0-OpenChat-7B](https://huggingface.co/beowolx/CodeNinja-1.0-OpenChat-7B)
+* [maywell/PiVoT-0.1-Starling-LM-RP](https://huggingface.co/maywell/PiVoT-0.1-Starling-LM-RP)
+* [WizardLM/WizardMath-7B-V1.1](https://huggingface.co/WizardLM/WizardMath-7B-V1.1)
+* [davidkim205/komt-mistral-7b-v1](https://huggingface.co/davidkim205/komt-mistral-7b-v1)
+* [OpenBuddy/openbuddy-zephyr-7b-v14.1](https://huggingface.co/OpenBuddy/openbuddy-zephyr-7b-v14.1)
+* [manishiitg/open-aditi-hi-v1](https://huggingface.co/manishiitg/open-aditi-hi-v1)
+* [VAGOsolutions/SauerkrautLM-7b-v1-mistral](https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-v1-mistral)
+
+## 🧩 Configuration
+
+```yamlbase_model: mlabonne/NeuralMarcoro14-7B
+dtype: bfloat16
+experts:
+- positive_prompts:
+  - chat
+  - assistant
+  - tell me
+  - explain
+  source_model: openchat/openchat-3.5-1210
+- positive_prompts:
+  - code
+  - python
+  - javascript
+  - programming
+  - algorithm
+  source_model: beowolx/CodeNinja-1.0-OpenChat-7B
+- positive_prompts:
+  - storywriting
+  - write
+  - scene
+  - story
+  - character
+  source_model: maywell/PiVoT-0.1-Starling-LM-RP
+- positive_prompts:
+  - reason
+  - math
+  - mathematics
+  - solve
+  - count
+  source_model: WizardLM/WizardMath-7B-V1.1
+- positive_prompts:
+  - korean
+  - answer in korean
+  - korea
+  source_model: davidkim205/komt-mistral-7b-v1
+- positive_prompts:
+  - chinese
+  - china
+  - answer in chinese
+  source_model: OpenBuddy/openbuddy-zephyr-7b-v14.1
+- positive_prompts:
+  - hindi
+  - india
+  - hindu
+  - answer in hindi
+  source_model: manishiitg/open-aditi-hi-v1
+- positive_prompts:
+  - german
+  - germany
+  - answer in german
+  - deutsch
+  source_model: VAGOsolutions/SauerkrautLM-7b-v1-mistral
+gate_mode: hidden
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers bitsandbytes accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "gagan3012/MetaModel_moe_multilingualv2"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "mlabonne/NeuralMarcoro14-7B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

model-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0adbacc08a9825c73631da8dbbb2a758b01090a9cb6aa7597f79c8e2136f0516
+size 9919813664

model-2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a034f3d253e5ff152d104649b819548e85415436ae7561fa2551bc55d811680
+size 9982454760

model-3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fc87aedde5bea472c14e7fe7d48f5717c745fc068d3dc50501925d9bd62514c
+size 9982454760

model-4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d14fdf959bb469a33e61c9ef560afe197cae1bbe07395d78e7f6bf4cd88dbbbc
+size 9982454680

model-5.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fd4200ee64034f3c8ba22683ffcc4783aca7ab3e475c737f6ffef7592b00190
+size 9982454760

model-6.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c083298e0ed5878fa0f990bc7bc7cd904baab3ffb4cf0f0a27e18769cfff2337
+size 9982454760

model-7.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:325fbc097fdfef5532373196c0c2462a9d0dd132c7ae39edf127e7b18c5717dd
+size 9982454680

model-8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9892556cf515c9cfebe2f1562bd7a6cabeae1a8063fcc2f1e2f315f1e3b69bd3
+size 9982454760

model-9.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd1656351f519ecb9af2a32e8748f8993a6301d4b7855499b94c284db72797ab
+size 9982454760