Upload folder using huggingface_hub

README.md

@@ -0,0 +1,55 @@
+---
+license: apache-2.0
+tags:
+- moe
+- frankenmoe
+- merge
+- mergekit
+- lazymergekit
+- johnsnowlabs/JSL-MedLlama-3-8B-v1.0
+- Weyaxi/Einstein-v6.1-Llama3-8B
+base_model:
+- johnsnowlabs/JSL-MedLlama-3-8B-v1.0
+- Weyaxi/Einstein-v6.1-Llama3-8B
+---
+
+# Llama3medical-15B-MoE
+
+Llama3medical-15B-MoE is a Mixture of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
+* [johnsnowlabs/JSL-MedLlama-3-8B-v1.0](https://huggingface.co/johnsnowlabs/JSL-MedLlama-3-8B-v1.0)
+* [Weyaxi/Einstein-v6.1-Llama3-8B](https://huggingface.co/Weyaxi/Einstein-v6.1-Llama3-8B)
+
+## 🧩 Configuration
+
+```yaml
+base_model: johnsnowlabs/JSL-MedLlama-3-8B-v1.0
+experts:
+  - source_model: johnsnowlabs/JSL-MedLlama-3-8B-v1.0
+    positive_prompts: ["medical"]
+  - source_model: Weyaxi/Einstein-v6.1-Llama3-8B
+    positive_prompts: ["what"]
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers bitsandbytes accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "allknowingroger/Llama3medical-15B-MoE"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "johnsnowlabs/JSL-MedLlama-3-8B-v1.0",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 2,
+  "output_router_logits": false,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.40.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

model-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78b612eb12fe39d08eb404c44f8d4b997522040a428d9409de4e56c577ad850a
+size 1050681576

model-10.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31a436fc9b004138ce2b726537fe12b654609d07285bf285162415eac015c0f0
+size 1996490936

model-11.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac6e2261de96fc841c1ece11abb4fae63d31e1dde6af5710a97d40994e28fa23
+size 1996490944

model-12.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905d40c9809896d8bfc10458d3aeb239fdba88fcbf05dc7fc4b07d4d8a594050
+size 1996490952

model-13.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a7fe98026fa29925f6c83b6058a11a9efffbbef962deb9d907c740d41a58a2
+size 1979980392

model-14.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1e4e3b71861b162ac66414863a9dd3c0e76d8fa82554b5b876d06742b3b487
+size 1979723896

model-2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceef2b3ab6b0769f794e860feff0d0cfc024e4fe23527e8fedfc38fbe30a42b3
+size 1990464104

model-3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb9e623f76b1535aa8b264527515c91fd31562eb06e6c847df729b8d87a9f8b7
+size 1996490936

model-4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69755cb07a64c18c6721db00c323d91284e4e6da37c41d3d7d5f5b541b8a7eaf
+size 1996490952

model-5.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11fef3834fa85aefdee9cf521b2f5edae6f13347c2e04dcaf4eec9afd1b958a1
+size 1996490952

model-6.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a71f4506a1abcbcff6db9f97aaec55300ce2be1c04c069b387d93243dca207c
+size 1996490936

model-7.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38cc8e097e4feb45a258ab0f787f8412dae87cfcf5b85c8ca51d4cb0594506d0
+size 1996490944

model-8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:415d361719a7eff9437a2ab34261546ae7c099a656a853a8ea79cec5db373ea1
+size 1996490952

model-9.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a5723c41b0c1c234e0b6f022bf1b58249b1accca1e32a5e7570e45915fde8f7
+size 1996490952