Upload 22 files

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "kyujinpy/Sakura-SOLAR-Instruct",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4096,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.37.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,67 @@
+base_model: kyujinpy/Sakura-SOLAR-Instruct
+gate_mode: hidden
+dtype: bfloat16
+experts:
+  - source_model: PracticeLLM/SOLAR-tail-10.7B-Merge-v1.0
+    positive_prompts:
+    - "helpful"
+    - "Relevant"
+    - "Factual"
+    - "Precise"
+  - source_model: NousResearch/Nous-Hermes-2-SOLAR-10.7B
+    positive_prompts:
+    - "Math"
+    - "Science"
+    negative_prompts:
+    - "inaccurate"
+    - "incorrect"
+  - source_model: kyujinpy/Sakura-SOLAR-Instruct
+    positive_prompts:
+    - "discuss"
+    - "chat"
+    - "culture"
+    - "world"
+    negative_prompts:
+    - "Sorry"
+    - "As an AI"
+    - "cannot"
+    - "not capable"
+  - source_model: NousResearch/Nous-Hermes-2-SOLAR-10.7B
+    positive_prompts:
+    - "calculate"
+    - "compute"
+    - "solve"
+    - "work"
+    negative_prompts:
+    - "mistake"
+    - "inaccurate"
+  - source_model: upstage/SOLAR-10.7B-Instruct-v1.0
+    positive_prompts:
+    - "assist"
+    - "logic"
+    - "sense"
+  - source_model: kodonho/SolarM-SakuraSolar-SLERP
+    positive_prompts:
+    - "form"
+    - "connect"
+    - "try"
+    negative_prompts:
+    - "cannot"
+    - "incapable"
+  - source_model: kyujinpy/Sakura-SOLAR-Instruct
+    positive_prompts:
+    - "percieve"
+    - "discern"
+    - "recognize"
+    negative_prompts:
+    - "don't"
+    - "cannot"
+  - source_model: kyujinpy/Sakura-SOLAR-Instruct
+    positive_prompts:
+    - "core"
+    - "common"
+    - "basic"
+    - "intuitive"
+    negative_prompts:
+    - "boring"
+    - "lifeless"

model-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:838942c2fe1e5565e61020018248d2df1306785957738862c924e9b5b1439dca
+size 9919946528

model-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:335821cc67c758adaf3463ce246f83a922c55b64885fb359883c56a17394dd56
+size 9982454760

model-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a41dd226dce3490c924981789ff464419ce2cff86e94ecb08b20bb0410cdb8e7
+size 9982454760

model-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78045917f54f2a2309c1642155c8c3dbd89bcd18e7885737ab2e33ca7d8ca322
+size 9982454760

model-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488b9b5de694a45058d6ad14135c6896524251e5d71121558b340a8833a70116
+size 9982454720

model-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b865e6431b9ba3bd59ddd3ef7cfaa44437f1cf115c01268475d1e2b8fc587de5
+size 9982454712

model-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3afda87fc2d72c840388cecbc03d243960b6817563f4318f4d9f40f27c1c50d9
+size 9982454760

model-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:154b6b039023319b2ea886dda6d3ad3f448cbf3049a32e09c79af6cc3ab14699
+size 9982454760

model-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35b4dadfd1b5b9c9c10055b192d9eb275ddbc8a4d0240c5c526cabf548cb6cd9
+size 9982454760

model-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfdd088ceaf77cb99245689245f11d9d572301dae0d47a512e9163c21773c85c
+size 9982454680

model-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7c30c57d124953df0518648584477319b50ef7cf6b0ee88a41c8ce8405d75e8
+size 9982454752

model-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:438b99565b16e0697a9c83691eb11c5bef19a9a4c8298265447a5ccfac00b9e2
+size 9982454760

model-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcedc43670d7673a7108a000346598352ee714492bab05ab3ba81b3b9ed69dad
+size 9982454760

model-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ca793714f2c1ffaf90ae0978b5014561d60b90172ca64d53e58f0a3be770cc5
+size 9999648648

model-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ce2083a134d058867894ab63830d7d0b82db07fbc978c7c0373bd2945c2d85
+size 137369408

special_tokens_map.json

@@ -0,0 +1,29 @@
+{
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,47 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n'  + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}