Upload folder using huggingface_hub

README.md

@@ -0,0 +1,102 @@
+---
+base_model:
+- m-a-p/neo_7b
+- m-a-p/neo_7b
+tags:
+- merge
+- mergekit
+- lazymergekit
+- m-a-p/neo_7b
+---
+
+# Neo_7b-merge2
+
+Neo_7b-merge2 is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
+* [m-a-p/neo_7b](https://huggingface.co/m-a-p/neo_7b)
+* [m-a-p/neo_7b](https://huggingface.co/m-a-p/neo_7b)
+
+## 🧩 Configuration
+
+```yaml
+slices:
+  # Group 1
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [0, 0]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [1, 1]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [2, 2]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  # Group 2
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [4, 4]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [5, 5]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [6, 6]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  # Continue this pattern for the remaining groups...
+  # Group 7 (last group)
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [24, 24]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [25, 25]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [26, 26]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+merge_method: slerp
+base_model: m-a-p/neo_7b
+parameters:
+  t: 0.3333  # Apply 1/3 of the 4th layer to each of the previous 3 layers
+dtype: bfloat16
+output_path: ./merged_redistributed_neo_7b
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "DewEfresh/Neo_7b-merge2"
+messages = [{"role": "user", "content": "What is a large language model?"}]
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
+
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "<|CLS|>": 64000,
+  "<|EOD|>": 64002,
+  "<|MASK|>": 64003,
+  "<|PAD|>": 64004,
+  "<|SEP|>": 64001
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "m-a-p/neo_7b",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 0,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 64256
+}

mergekit_config.yml

@@ -0,0 +1,57 @@
+
+slices:
+  # Group 1
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [0, 0]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [1, 1]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [2, 2]
+      - model: m-a-p/neo_7b
+        layer_range: [3, 3]
+  # Group 2
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [4, 4]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [5, 5]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [6, 6]
+      - model: m-a-p/neo_7b
+        layer_range: [7, 7]
+  # Continue this pattern for the remaining groups...
+  # Group 7 (last group)
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [24, 24]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [25, 25]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+  - sources:
+      - model: m-a-p/neo_7b
+        layer_range: [26, 26]
+      - model: m-a-p/neo_7b
+        layer_range: [27, 27]
+merge_method: slerp
+base_model: m-a-p/neo_7b
+parameters:
+  t: 0.3333  # Apply 1/3 of the 4th layer to each of the previous 3 layers
+dtype: bfloat16
+output_path: ./merged_redistributed_neo_7b

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41209159383a6438fa1e73146df470aae582c40bf510ec2495a3f8780477ce87
+size 789584192

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93f2a7f3db85521671732789bb8fd1bb3ae3a7e0d33170ea530f53a25346fcdd
+size 4998668592

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1227a57ac0fc74f5a2c223e5e41dd53ba09f50acbf6d63e5f166a8e1fc740c5e
+size 4926336584

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5abc28b3d1b89d4077e831f565ea2b0794a20ade31c750e450a00c7ebc9327ea
+size 4907455792

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:028abab236ed774f37153fc8d7c0dc2f87b34b3dc964d20ffab314a43f3d6706
+size 226505520

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.4", "total_size": 789583872}, "weight_map": {"lm_head.weight": "model-00001-of-00001.safetensors", "model.embed_tokens.weight": "model-00001-of-00001.safetensors", "model.norm.weight": "model-00001-of-00001.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "additional_special_tokens": [
+    "<|CLS|>",
+    "<|SEP|>",
+    "<|EOD|>",
+    "<|MASK|>",
+    "<|PAD|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6a2447b0e5664cabb2481587597102d82f42f0ccb7ef22e1c2d95494a8b03c5
+size 1002561

tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    },
+    "64000": {
+      "content": "<|CLS|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "<|SEP|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64002": {
+      "content": "<|EOD|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64003": {
+      "content": "<|MASK|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64004": {
+      "content": "<|PAD|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|CLS|>",
+    "<|SEP|>",
+    "<|EOD|>",
+    "<|MASK|>",
+    "<|PAD|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "m-a-p/neo_7b--tokenization_neo.NEOTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% set system_message = 'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\\n\\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don\\'t know the answer to a question, please don\\'t share false information.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '<s>' + '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 4096,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "NEOTokenizer",
+  "unk_token": "<unk>",
+  "use_fast": false
+}