Upload folder using huggingface_hub

config.json

@@ -10,7 +10,7 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 32768,
   "model_type": "mixtral",
   "num_attention_heads": 32,
   "num_experts_per_tok": 2,
@@ -24,7 +24,7 @@
   "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.36.2",
+  "transformers_version": "4.37.1",
   "use_cache": true,
   "vocab_size": 32000
 }

mergekit_moe_config.yml

@@ -0,0 +1,31 @@
+
+base_model: mlabonne/Marcoro14-7B-slerp
+experts:
+  - source_model: openchat/openchat-3.5-1210
+    positive_prompts:
+    - "chat"
+    - "assistant"
+    - "tell me"
+    - "explain"
+  - source_model: beowolx/CodeNinja-1.0-OpenChat-7B
+    positive_prompts:
+    - "code"
+    - "python"
+    - "javascript"
+    - "programming"
+    - "algorithm"
+  - source_model: maywell/PiVoT-0.1-Starling-LM-RP
+    positive_prompts:
+    - "storywriting"
+    - "write"
+    - "scene"
+    - "story"
+    - "character"
+  - source_model: WizardLM/WizardMath-7B-V1.1
+    positive_prompts:
+    - "reason"
+    - "math"
+    - "mathematics"
+    - "solve"
+    - "count"
+tokenizer_source: union

model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79a84b6fdb3ece83c2703ab2de0a3333aeeafd213b872d82a8261de61e63429c
+oid sha256:f1e3cc831230c664a8590c85633fccc26a93e51a0bcfb8ea3dd9039c22847632
 size 8440279464

tokenizer_config.json

@@ -36,12 +36,11 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
-  "model_max_length": 8192,
+  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<s>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + eos_token}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
   "use_default_system_prompt": true
-}
+}