mergekit_moe_config.yml

base_model: meta-llama/Llama-2-7b-chat-hf
gate_mode: random
dtype: bfloat16
experts_per_token: 2
experts:
  - source_model: meta-llama/Llama-2-7b-hf
    positive_prompts:
      - "should be able to converse properly"
    negative_prompts:
      - "Uncensored in my opinion"
      
  - source_model: syzymon/long_llama_code_7b_instruct
    positive_prompts:
      - "Perform pretty well in coding question"
      
    negative_prompts:
      - "Is quite bad in C++"
      
  - source_model: georgesung/llama2_7b_chat_uncensored
    positive_prompts:
      - "Uncensored"
      
    negative_prompts:
      - "really bad in high school grade math and science"
      
  - source_model: togethercomputer/LLaMA-2-7B-32K
    positive_prompts:
      - "really good in long context question answering"  
      
    negative_prompts:
      - "incorrect or biased content"