Upload folder using huggingface_hub

README.md

@@ -6,43 +6,66 @@ tags:
 - merge
 
 ---
-# MN-Three-RCM-Instruct1-2a
+# Nemo-Grand-Gutenburg3
 
 This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
 
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [DARE](https://arxiv.org/abs/2311.03099) [TIES](https://arxiv.org/abs/2306.01708) merge method using E:/MN-Rocinante-12B-v1.1-Instruct as a base.
+This model was merged using the passthrough merge method.
 
 ### Models Merged
 
 The following models were included in the merge:
-* E:/MN-magnum-v2.5-12b-kto-Instruct
-* E:/MN-12B-Celeste-V1.9-Instruct
+* G:/11B/Lyra4-Gutenberg-12B
+* G:/11B/Lyra-Gutenberg-mistral-nemo-12B
+* G:/11B/mistral-nemo-gutenberg-12B-v4
 
 ### Configuration
 
 The following YAML configuration was used to produce this model:
 
 ```yaml
-# Config 1
-# E:/MN-Rocinante-12B-v1.1-Instruct
-# E:/MN-12B-Celeste-V1.9-Instruct
-# E:/MN-magnum-v2.5-12b-kto-Instruct
-
-models:
-  - model: E:/MN-Rocinante-12B-v1.1-Instruct
-  - model: E:/MN-magnum-v2.5-12b-kto-Instruct
-    parameters:
-      weight: .6
-      density: .8
-  - model: E:/MN-12B-Celeste-V1.9-Instruct
-    parameters:
-      weight: .38
-      density: .6
-merge_method: dare_ties
-tokenizer_source: union
-base_model: E:/MN-Rocinante-12B-v1.1-Instruct
+# NEMO-Mistral version -> Scaled to 40 layers.
+
+# fix one -> 
+# Fix 3 point double match @ layers 14,15 all 3 models (remove 2 layers)
+
+# fix 2 ?? - restored fix 1, 
+# 30-31 -> all models 2nd group.
+
+# Fix 3??
+# 36-39 -> second last block.
+
+slices:
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [0, 16] 
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [12, 24] 
+ - sources:
+   - model: G:/11B/Lyra-Gutenberg-mistral-nemo-12B
+     layer_range: [14, 28] 
+
+# secondary block
+
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [18, 30] # remove layer 30
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [31, 32] # remove layer 30
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [24, 31] # remove layer 31
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [32, 39] # remove layer 31
+ - sources:
+   - model: G:/11B/Lyra-Gutenberg-mistral-nemo-12B
+     layer_range: [30, 40] 
+merge_method: passthrough
 dtype: bfloat16
 ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "E:/MN-Rocinante-12B-v1.1-Instruct",
+  "_name_or_path": "G:/11B/Lyra4-Gutenberg-12B",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -14,7 +14,7 @@
   "max_position_embeddings": 1024000,
   "model_type": "mistral",
   "num_attention_heads": 32,
-  "num_hidden_layers": 63,
+  "num_hidden_layers": 79,
   "num_key_value_heads": 8,
   "rms_norm_eps": 1e-05,
   "rope_theta": 1000000.0,
@@ -22,6 +22,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.43.3",
-  "use_cache": true,
-  "vocab_size": 131074
+  "use_cache": false,
+  "vocab_size": 131072
 }

mergekit_config.yml

@@ -1,19 +1,41 @@
-# Config 1
-# E:/MN-Rocinante-12B-v1.1-Instruct
-# E:/MN-12B-Celeste-V1.9-Instruct
-# E:/MN-magnum-v2.5-12b-kto-Instruct
+# NEMO-Mistral version -> Scaled to 40 layers.
 
-models:
-  - model: E:/MN-Rocinante-12B-v1.1-Instruct
-  - model: E:/MN-magnum-v2.5-12b-kto-Instruct
-    parameters:
-      weight: .6
-      density: .8
-  - model: E:/MN-12B-Celeste-V1.9-Instruct
-    parameters:
-      weight: .38
-      density: .6
-merge_method: dare_ties
-tokenizer_source: union
-base_model: E:/MN-Rocinante-12B-v1.1-Instruct
+# fix one -> 
+# Fix 3 point double match @ layers 14,15 all 3 models (remove 2 layers)
+
+# fix 2 ?? - restored fix 1, 
+# 30-31 -> all models 2nd group.
+
+# Fix 3??
+# 36-39 -> second last block.
+
+slices:
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [0, 16] 
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [12, 24] 
+ - sources:
+   - model: G:/11B/Lyra-Gutenberg-mistral-nemo-12B
+     layer_range: [14, 28] 
+
+# secondary block
+
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [18, 30] # remove layer 30
+ - sources:
+   - model: G:/11B/mistral-nemo-gutenberg-12B-v4
+     layer_range: [31, 32] # remove layer 30
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [24, 31] # remove layer 31
+ - sources:
+   - model: G:/11B/Lyra4-Gutenberg-12B
+     layer_range: [32, 39] # remove layer 31
+ - sources:
+   - model: G:/11B/Lyra-Gutenberg-mistral-nemo-12B
+     layer_range: [30, 40] 
+merge_method: passthrough
 dtype: bfloat16

special_tokens_map.json

@@ -1,23 +1,30 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}