Upload folder using huggingface_hub (#1)

- d074544d3562ffb339b5a40eb9ab0b489a23d39fc76956e23afe69670027bf90 (7ac31c6376683669aa13ec9371b2ba9926cc8393)
- 268ee4cecfb5cdd0aef4e626539b0c0d6b139435957d550cd4d6ac9032b9fe56 (6332b4c6ce56a87d714496be7e28900b6fee9c13)
- 6d0047d43069ab4ad9d93bda735deb2940f5e1e820c8db12145192891dd3dfc7 (40a0f439ba73394214e346552e5647ef591d73a5)
- 78a19d0d43fac13ca6c5cb27aaf37c55f39326e00a51a0be6ad0c7c44850e80b (dde12b8b9d690ac0c659ada831b0e567a478d8ff)
- bfea9f76e3c614dbea62928028ea1e4252c87c31591c50088d8b78e80eb0ddf5 (3bc0a9d1c178a140a6f34c88125d4485664353c1)
- 8d79514772ff90d340dc8d61239721f84f58d585d8ad790a92c9eea0a50084de (b52a1a339dc2d873b491be99a48648de9097ffa5)
- 81cac7bb8b4f2ce0fd71df0a5f85e0e3b6c3120d40f0dd8484591b8feba2fd9c (5141540dafce78c4586e443110dd25d61f20adc3)
- 4b07d9505fabeac114423aefdfdeae62b0af0119d6a4e32d4a63a6d9c80c8bc0 (60c4b34dad0ef91c8fe51c83bda40387f02a8267)
- f2b011af4aa0bdb14a809ae5e2440032dd5bb82f40cdd7f785a76d46f0e7c293 (e922351420311d6d347bb0c0764b9840735a164d)
- 15fd7c742e89c2b5db2b0f566eb0cd873e6fba362f8645e555e4297fb3fe9c1e (16959fa999854ba0ab6fb0a2b631187fe9cc5967)
- f3183980068c5ad6a774918d84181a35fd0748f513dcaa3f2872bed5bae4d963 (c24734683d36bcc8976eb30b3c0cd2c3e3b50237)
- 8eaede89853235440d684137d210c12a5182ca84d8ea1e9ac5037c8659f283c3 (da9321d95f40f565fae509b4067ae3db785a064f)
- 99ea133c5d351abb6920b24ce9b18f112a7ac7767cd1279a778f62dcf4eed596 (82eff9293e43bf0d055f5c5f9c7167a2e723f970)
- 3e79e0d4c005ce36c355ccecf772278a5dc8c50376c4201c97753bf7931f8246 (100b3c937caa371c6b341038903703099a77308d)
- e2915e9889fd4665d95af904a91940439e34382aa0d612d07baaca993ed508c2 (e2978aa2018324a8a96eb7e2d052807684c02a04)
- 2c1751accdcd936962592ac1f619a3996c3a8e99297baa4f2ab889683a25dab2 (afb26873a4dc32a4c9e410b000f5517ffdf635df)
- d82c8b5d65931f4dcb230106c6ff62fdd39a7990551bdd0a85ed5868ba38c01a (1c67e5dee65ea641714c79cb81ace79d408938d9)
- 1d1555c919d7ffb54880ecd60c98eec88b449d4fcc452714a3742fb46955da11 (3d8f92132875a2996dd8049f4e4464f70cb0368d)
- 981772118c66c61aa1628f1db0c6f07aeb650b4105a93b7f245dafb6a3e8d4c1 (ccda3df194057695419c092ae4f14c99d1afe6ac)
- ee59dfd8a4fd7ee4897307b356fde53034556efd5c27a87e43ae9f7963e4b3a3 (92b21ecf68d3d4dc303b44288b4fc194c83eb0e7)
- 7558e9aada530fa6ed7d14343735061b7c5efd694f1ec52e5f1c59b6e7025327 (bbfb97c6847b52d6f4e4f09bccee15b43cbce284)
- a69b82c536c9544d7365a83b076885368650060df4dceafb1faf37808c73df01 (a731ebd881f78d70116792a62d1f8886d34c6587)
- 75bf430b75ae0ff2fcda4293dd4a8c9ae07df7577f1dbf6da46d5ba12b3ada98 (cd2e8f3f35a0f23a9500772a5f708c51f2e23284)
- b75ba965b7ebd4534786dafbc8ed997fea9f89afbf53bcd87976a40044f23a45 (62b9bebc086138f985ac5163f4aab20bca51652d)
- bfe03017b500fa8eff0714ba7caff1f25e46cd4db250e46f3438d5b0d19f73ad (c16a22af23c13509ca4a0a4b0c6a6da6e34cfbe4)
- 2be2bd95803cd97c4ed38fce993fd72f9055ca9a6a1c38eef19bf235d66d0fc2 (3ca45f997ffd429de47348f8401e992c41164efe)
- 01ad15f414396b3ef5bc845cafd0858f1582662427e710c3d7140b91ba687254 (fe0229bf1085900ae6da8c9aaa5c8138f9828063)
- 962686115f3390c2678e9cbef8224e0a239178d685c3ed19f5bf854e16dba132 (acb7057ff32946997b48e09f30e89ab87d3fb810)
- 607917d6001e58d0b3f9892b8f52717b980beb6041bef01a09977b0507edcc60 (eb3df9044a1cedd39aa09e571cc5ed9cdc77c6f2)
- ced3b567f2d0a6c8a375210df683c24c19b4b09180a29df5e1e4f62ee70deed2 (68fc02c8017e1c5ce9b120b0ce638d5d7f22e1a3)

.ipynb_checkpoints/README-checkpoint.md

@@ -0,0 +1,48 @@
+---
+base_model:
+- alpindale/WizardLM-2-8x22B
+- mistralai/Mixtral-8x22B-Instruct-v0.1
+tags:
+- mergekit
+- merge
+
+---
+# merge.py
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B)
+* [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+
+slices:
+  - sources:
+      - model: mistralai/Mixtral-8x22B-Instruct-v0.1
+        layer_range: [0, 56]
+      - model: alpindale/WizardLM-2-8x22B
+        layer_range: [0, 56]
+merge_method: slerp
+base_model: mistralai/Mixtral-8x22B-Instruct-v0.1
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+dtype: bfloat16
+
+```

.ipynb_checkpoints/config-checkpoint.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "mistralai/Mixtral-8x22B-Instruct-v0.1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "use_cache": true,
+  "vocab_size": 32768
+}

.ipynb_checkpoints/mergekit_config-checkpoint.yml

@@ -0,0 +1,17 @@
+
+slices:
+  - sources:
+      - model: mistralai/Mixtral-8x22B-Instruct-v0.1
+        layer_range: [0, 56]
+      - model: alpindale/WizardLM-2-8x22B
+        layer_range: [0, 56]
+merge_method: slerp
+base_model: mistralai/Mixtral-8x22B-Instruct-v0.1
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+dtype: bfloat16

README.md

@@ -0,0 +1,48 @@
+---
+base_model:
+- alpindale/WizardLM-2-8x22B
+- mistralai/Mixtral-8x22B-Instruct-v0.1
+tags:
+- mergekit
+- merge
+
+---
+# merge.py
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B)
+* [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+
+slices:
+  - sources:
+      - model: mistralai/Mixtral-8x22B-Instruct-v0.1
+        layer_range: [0, 56]
+      - model: alpindale/WizardLM-2-8x22B
+        layer_range: [0, 56]
+merge_method: slerp
+base_model: mistralai/Mixtral-8x22B-Instruct-v0.1
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+dtype: bfloat16
+
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "mistralai/Mixtral-8x22B-Instruct-v0.1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "use_cache": true,
+  "vocab_size": 32768
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+
+slices:
+  - sources:
+      - model: mistralai/Mixtral-8x22B-Instruct-v0.1
+        layer_range: [0, 56]
+      - model: alpindale/WizardLM-2-8x22B
+        layer_range: [0, 56]
+merge_method: slerp
+base_model: mistralai/Mixtral-8x22B-Instruct-v0.1
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+dtype: bfloat16

model-00001-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8567b571939ae5dfb5d48c49b16563253dc36e69cf117e0e7df3c1d2ef6e1a
+size 9805561216

model-00002-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35da0fa5000f7e5f08eeec69b68bd2660a8862e1f60ded9f7f79823310538301
+size 9814924936

model-00003-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e87dd44a050a913508f46c78617264149c6c3c47fbcecbd58a25fb4aa7824fd
+size 9814924936

model-00004-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b89941a6ec1fe3c6173b3e36fd489da7274fe0b52b7902a57aed087a4b1847
+size 9814924936

model-00005-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6dff6be8d77dedaa4d31f32ae303c541f7a886505c12fad9397d2c905834f5
+size 9814924944

model-00006-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8624892ac9044964c875b06c2db7c91745393a0a290874358d97c343d4db31ec
+size 9814924984

model-00007-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a46d253d6919362f1ae2927bf19840ba814210a8bf27785579ade33a2c22bbc1
+size 9991086240

model-00008-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e55b5f572ff519aadb1accec22ac4050842b0c9cb03c216affe3f56985197b41
+size 9814925000

model-00009-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a9ca4a8ceff1b6970da8b2aed31030efcd0084fee696308a2289d21a8c50311
+size 9814925000

model-00010-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2450a798bcac95c61b49766c317329352db1ee9725003193fb27981bb8cc70c
+size 9814925000

model-00011-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1069ffd8bd4db1ad5bdf344048ae8fac17aef5d0c110ec043f48f57cda65c913
+size 9940655712

model-00012-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5e260cb350e31366b700bd76e22f9369caefa1a2484275c64785c0b55215736
+size 9903005048

model-00013-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d5da31c3d475f3276ba8f1bbddee5e3f117879bb11a92ea04e2786ff89b16b1
+size 9852673512

model-00014-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4302d1e5cda294fc68265486372af50246345bcd1ed6a471fd217bda73be1b
+size 9814900192

model-00015-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0862d1164d0266502a32f41fa8a85388dc82d7d4e132f4a9a1167710752620cd
+size 9814925000

model-00016-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0521ccafb1043fdf2a1c8eebc493914feb0d204e88aa213b071f4691fae4cb03
+size 9814925000

model-00017-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1beffd9064e76c73b2993f9767544812a6a9b6b035dbb5cb7a8a46aeaf3bbb46
+size 9991086240

model-00018-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b2b3f60f4a9eb4da537ff1198caa8d07dd9d4c6bc0ad904fe7d767de01c5fa
+size 9814925000

model-00019-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a1974226df47d63d8ebac3036baf463a325ba933576201aec4a8416d5292232
+size 9814925000

model-00020-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3dc866ff028b49ab85fa20ebf15812b0cc2b4ad2e3690d75120af530da747da
+size 9814925000

model-00021-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be418005d1dedfc40b44af4087a6f5288a6bedd02427b34f7e4e7574f990e352
+size 9814925000

model-00022-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6f3b58bbadb2bdf320cc04b75f23fa7d0da01b64e1c53bada73090e4e3f41c2
+size 9814925000

model-00023-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d41167ae5d1c97049642000b60b8a5cb4781f8af6618d80602d40f2ab2a558d
+size 9814925000

model-00024-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0586c1baf7eb695b0866b3000c26963f855f587eb7b10ad4f23840e6fd8a81a5
+size 9991209472

model-00025-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d14c1b8276a62786f3f6987db8035453f862ad6832705ab5db9e3a1d48105b2c
+size 9852575088

model-00026-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0395ea255b66391c1278012a323d36ed7e63ecb1d2d31ef23f9d32dbafe9b00c
+size 9903005048

model-00027-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7db19d71c9dcd7ea6f4c5b2f067fe5ed9be0b321ae42478a26272ddf36e0d610
+size 9940729328

model-00028-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0d2bdb93a12602e3a7c06c999f380485ff8f3ac3cedc1396351398881e4b355
+size 9814925000

model-00029-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ab71c0185b7a860b4764439440202e7ab67883b7c770c9d21ce2032ab18a6f
+size 5401280728

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,99 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[INST]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[/INST]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[TOOL_CALLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "[AVAILABLE_TOOLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "[/AVAILABLE_TOOLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "[TOOL_RESULTS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "[/TOOL_RESULTS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{bos_token}}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}