Upload folder using huggingface_hub

DolphinProf.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": "."
+		},
+		{
+			"path": "../../git/mergekit"
+		}
+	],
+	"settings": {}
+}

README.md

@@ -81,7 +81,7 @@ dtype: float16
 tokenizer_source: model:cognitivecomputations/dolphin-2.2-70b # keep exact tokenizer used by dolphin - or you could use `union` if you add all of the input models to the first/last slice, but they would need to be non-zero weight or you'll get NaNs in your embeddings
 ```
 
-# Example Output  (Generated with 4-bit gguf quantized version)
+# Example Output
 
 > Please invent a new idea in the area of mathematics, that combines two or more papers into a new idea that has not yet been published to your knowledge
 ## Integrability and Computability of Continuous Functions on Infinite Domains: A Category-Theoretical Approach

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32000,
+  "<|im_start|>": 32001
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "cognitivecomputations/dolphin-2.2-70b",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 180,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32002
+}

mergekit_config.yml

@@ -0,0 +1,47 @@
+merge_method: linear # use linear so we can include multiple models, albeit at a zero weight
+parameters:
+  weight: 1.0 # weight everything as 1 unless specified otherwise - linear with one model weighted at 1 is a no-op like passthrough
+slices:
+  - sources:
+      - model: cognitivecomputations/dolphin-2.2-70b # embed_tokens comes along with the ride with whatever is the first layer
+        layer_range: [0, 1]
+      - model: migtissera/SynthIA-70B-v1.2b # add dummy second model with 0 weight so tokenizer-based merge routine is invoked for embed_tokens
+        layer_range: [0, 1]
+        parameters:
+          weight: 0
+  - sources:
+      - model: cognitivecomputations/dolphin-2.2-70b
+        layer_range: [1, 20]
+  - sources:
+      - model: migtissera/SynthIA-70B-v1.2b
+        layer_range: [10, 30]
+  - sources:
+      - model: WizardLM/WizardMath-70B-V1.0
+        layer_range: [20, 40]
+  - sources:
+      - model: epfl-llm/meditron-70b
+        layer_range: [25, 45]
+  - sources:
+      - model: cognitivecomputations/dolphin-2.2-70b
+        layer_range: [30, 50]
+  - sources:
+      - model: migtissera/SynthIA-70B-v1.2b
+        layer_range: [40, 60]
+  - sources:
+      - model: WizardLM/WizardMath-70B-V1.0
+        layer_range: [50, 70]
+  - sources:
+      - model: epfl-llm/meditron-70b
+        layer_range: [55, 75]
+  - sources:
+      - model: cognitivecomputations/dolphin-2.2-70b
+        layer_range: [60, 79]
+  - sources: # same as above, but for lm_head with the last layer
+      - model: cognitivecomputations/dolphin-2.2-70b
+        layer_range: [79, 80]
+      - model: migtissera/SynthIA-70B-v1.2b
+        layer_range: [79, 80]
+        parameters:
+          weight: 0
+dtype: float16
+tokenizer_source: model:cognitivecomputations/dolphin-2.2-70b # keep exact tokenizer used by dolphin - or you could use `union` if you add all of the input models to the first/last slice, but they would need to be non-zero weight or you'll get NaNs in your embeddings

model-00001-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc1f515376643696b5509f8c984f6557a4f50117a2bc8e532a36bbee2fd738f1
+size 9986840424

model-00002-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d975a85c77ed7aebba0df45caa990c725c58db0fadd781fed5d43cc8c66dcc95
+size 9798081088

model-00003-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e347cad10be6ebcbb3273abe6a4e614ddc806a8388069750cda86698489fa87d
+size 9965852200

model-00004-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:193eeb23b10ccbf5d2befaea8c0cc891d257a913e4226a57de3194f299d49277
+size 9798080424

model-00005-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7093ed30bca6efcedd46f076215aa0c7c876c9b44f6f2bb98120fd04625e942
+size 9965868720

model-00006-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bceceb6a3b836e06656f23a715c1a3454a77170c049522301e85055f39225179
+size 9965868720

model-00007-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a4e533dfd31f8cb4cccf0d569c0945962cd46a5e1133e5da8a12f8dfe4a87f9
+size 9798096912

model-00008-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcacf3df594becf74fdd69d95e03a3429b31ef1a06752263977e840ccda33c70
+size 9798096896

model-00009-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df5a71dff2d98ce1c34f7b485f9d98960c0ec37e92d4efc72665da7d45a2c744
+size 9965852216

model-00010-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93f5421e951030b5922a1bdc1b75b30254176be43d529a28b88189b8b65ff1de
+size 9630275624

model-00011-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54323e93a709646377f75b9dcbae9c385610fb80631096bf31792619259ccae2
+size 9798096936

model-00012-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f471aa32b901dcc790cb03a4e0c207cfd867a6a5697172837c76732b594ca34
+size 9798096952

model-00013-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4f96305e454995b053b83ae709e915a17025b1e7c65d58875795c98a399e123
+size 9798096920

model-00014-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:150770b4314788a1b4898a2fa7fb7353fd2d9d67657b6796aeda2252c10d3d76
+size 9965868752

model-00015-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:320e9c280b375d4010c21e3552b17efadc87c2697e7f4b3ab750ee95a259452e
+size 9965885240

model-00016-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a786826334cf4cbe07c702e1e2a0a510b82a186027e23689faa9aefd15bd23
+size 9798080424

model-00017-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c56e77028142b50084eb3a40cd3d25c4d3259fa1f604a55755032e87490f002
+size 9798063944

model-00018-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fef28e5d83623d0e8cf1ae02dc676b9e48c581a098a89456d1627009f5e4be8b
+size 9663846072

model-00019-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8161098b800cbc962bc5f2df1a40288b7cd51227ba193f553b3ced0e57890641
+size 9798096952

model-00020-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de924bb285bf6fa16525f329f6511a6c4f2ce3959790c750784729b717cc1abe
+size 9798129904

model-00021-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b19593608d51b4968eb21984f07ee54804b4c32a226acabda22ca4ab8fa68be
+size 9630291448

model-00022-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d54fe3a7a938ae6a810485fe57b9d3e68071ce1a22d869833f13515bc832a07
+size 9798096936

model-00023-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:977577794d614156f1ae932c824ae47935fb26f8a7b913d66e5fcd5c27a86ec9
+size 9965835752

model-00024-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e17c6b62353b784d2e2f8f8d6963a24a79bde5507d4bf02dc618b6156e14ecd7
+size 9798096952

model-00025-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1a4297e6519fa8af018c446d32ee206d06f558336ff6c23688f62303095a724
+size 9965852944

model-00026-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01db6dcb0b659323e8a28171a79ca25174ccd83c7756c5231fa49907cae77176
+size 9798113448

model-00027-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74fdc329c94d413cdb28953d036693a134f033b1b57c35f760e2c6df8e3c20f1
+size 9965885232

model-00028-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085a0d37413e40bc61acdfb2fb2f2ddfd39a7661eb950ba62d3dfc2044c49b19
+size 9550665264

model-00029-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38553cc54e82a00fa231b3356339e5f0058a1878d38150938729d3216a3ef58a
+size 9630274944

model-00030-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cef9fa248f4468f2402225c0b4ab4d03b5c19ca8db75ebd8281850f7cd93684
+size 9965885976

model-00031-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e8e9db59eb0ef2aa207e34d4fc85e0c0123030e27c08c6de29154ca58dbff8b
+size 9798079744

model-00032-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed036ef34236715773ce4bdd4712af2f2114c2308544e3d5c8dbd7e46f3c1d43
+size 4362177048

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": false,
+  "max_length": 2048,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "stride": 0,
+  "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true,
+  "use_fast": true
+}