Upload folder using huggingface_hub

README.md

@@ -4,18 +4,17 @@ tags:
 - mergekit
 - lazymergekit
 - allknowingroger/Multimerge-12B-MoE
-- zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0
+- TomGrc/FusionNet_7Bx2_MoE_v0.1
 base_model:
 - allknowingroger/Multimerge-12B-MoE
-- zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0
-license: apache-2.0
+- TomGrc/FusionNet_7Bx2_MoE_v0.1
 ---
 
 # Multimerge3-12B-slerp
 
 Multimerge3-12B-slerp is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
 * [allknowingroger/Multimerge-12B-MoE](https://huggingface.co/allknowingroger/Multimerge-12B-MoE)
-* [zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0](https://huggingface.co/zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0)
+* [TomGrc/FusionNet_7Bx2_MoE_v0.1](https://huggingface.co/TomGrc/FusionNet_7Bx2_MoE_v0.1)
 
 ## 🧩 Configuration
 
@@ -24,7 +23,7 @@ slices:
   - sources:
       - model: allknowingroger/Multimerge-12B-MoE
         layer_range: [0, 32]
-      - model: zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0
+      - model: TomGrc/FusionNet_7Bx2_MoE_v0.1
         layer_range: [0, 32]
 merge_method: slerp
 base_model: allknowingroger/Multimerge-12B-MoE
@@ -35,7 +34,7 @@ parameters:
     - filter: mlp
       value: [1, 0.5, 0.7, 0.3, 0]
     - value: 0.5
-dtype: bfloat16
+dtype: float16
 ```
 
 ## 💻 Usage

config.json

@@ -24,7 +24,7 @@
   "router_jitter_noise": 0.0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.40.2",
   "use_cache": true,
   "vocab_size": 32000

mergekit_config.yml

@@ -3,7 +3,7 @@ slices:
   - sources:
       - model: allknowingroger/Multimerge-12B-MoE
         layer_range: [0, 32]
-      - model: zhengr/MixTAO-7Bx2-MoE-Instruct-v7.0
+      - model: TomGrc/FusionNet_7Bx2_MoE_v0.1
         layer_range: [0, 32]
 merge_method: slerp
 base_model: allknowingroger/Multimerge-12B-MoE
@@ -14,4 +14,4 @@ parameters:
     - filter: mlp
       value: [1, 0.5, 0.7, 0.3, 0]
     - value: 0.5
-dtype: bfloat16
+dtype: float16

model-00001-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b0d2dae899289533247a40c54c0fd4fe73ad71bb2acec9922cd9bd2cb139217
+oid sha256:b666f336c8d537c98641781dfe080153df45cc0153b3135ab7d8c72351de4e32
 size 994050792

model-00002-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4da7b98794f6c36357785dfdd3e0d6a4484156ea93b617c071c2082d0cf6bc6
-size 906004176
+oid sha256:ba8284fa6a11c667ead6276a1279483390860d4e3eb3a1befb484b7cabadf29c
+size 906004160

model-00003-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30c04403779b115f608be591fa81ad5018a76b90e430c493022af9f5351b19f4
-size 989923840
+oid sha256:2cd3f77361ab998eba65afa455793ac52261c78dcce2bc9b9551ba8ef1f41c05
+size 989923816

model-00004-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fea16e206fd2c06133fe170f74ef436c3ae359c1fdec2ac63a287f35bd6791e5
-size 906004192
+oid sha256:71337c1b617fbd3aedc0eb711442b2e187d6bf39d2733d7c697e03f6475cd0a1
+size 906004176

model-00005-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:daccd1f1b2a50d3658391e248b82ac13820306850a74a8bec77d9e1110fd1f38
-size 906004192
+oid sha256:f45005a7a058b433d5bd5b9e255031e8482a08c96909c9aea7cef6e28fd3ac68
+size 906004176

model-00006-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3a6a95a71fca80e3cbf3eb8a453e87d351fc4697a92c2c1bf240d2936a88768
-size 906004192
+oid sha256:8a1c15cda0c3fbac94eff014a857ee0812d5deecb7f50cd3d7d30208e2f8306b
+size 906004176

model-00007-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a552dac23575dbc0c5df9a7db2667b76b859f0e7650fe6662aa8ab4a46b686c
-size 906004192
+oid sha256:2c809bd961ba970627ab7c372f019a6f82d7138f567a124ab488c38b1a051c6f
+size 906004176

model-00008-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52f783bd540fbb6e7f0ed6061909a9d0e64fbb150c5c27a5377c018b4889ee55
-size 906004192
+oid sha256:4cc953a34ba548d24bef1401b450ac9a2a928a3d229f2ec744585bc86d123012
+size 906004176

model-00009-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6b86b8e08d290e7a8e995763a98c940e9d82052dc6713e59be71c850b3e1fdf
-size 989923848
+oid sha256:2a418785896da9adfcfcd121fdb41034883aa736da5a3db930b8ae3ca9bb259e
+size 989923824

model-00010-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:618a826a93d77a6422f56834eba1b0dc275a476274e11c4f00284a5cc210effe
-size 906004192
+oid sha256:f238e08d2f6f7fe5fcecd77906979d374c6707994ad7504c8d22f8a2a2b38efd
+size 906004176

model-00011-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73850acd4032a8f938bf00f743bd52e566ab09cce39f9b554d3000b9f2e6b57e
-size 906004184
+oid sha256:5d1cadaa4905d1ecb0ddff58337b75ebaa8cd7f27bdfa08eebabcf71dc1a67bc
+size 906004176

model-00012-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f3bf9191974c4aee73ade25ef0889824f14c98925adee1e71d6fb4669906789
-size 906004176
+oid sha256:dad78558145977c3e2612cdede30415cf88bf76e108378f95b2a13280f469d4b
+size 906004160

model-00013-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c86f98e19103462519f06b05784ae87a10b70c2127aca5e62a2603d773799aa7
-size 906004192
+oid sha256:f5888703ce0293e588d59544dc87e70bbf8f091b9cf103fb8831d875406b9b87
+size 906004176

model-00014-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d1daba4f1ad25141bde152018875394aabbe1eedaa81dd42a4e15db47653da9
-size 906004192
+oid sha256:06c789f799e9736998e5263916538ac499ecbd9f201348beac2939151430aa03
+size 906004176

model-00015-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f22adbfa09b48757f667ab7d26cb8d432133046c917473454a8af3962419ad3
-size 989923848
+oid sha256:efa59eecdbdaba5a18be8f971d12a27dbcc31c52e9b6430d750dfc6417c13dc6
+size 989923824

model-00016-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58b782886b35253a5b8d7b270bebbb2235c3d7db3adc82fab576c333862a545a
-size 906004192
+oid sha256:23cfc7ef27bef1c78a9e9718d47c3fe1afa9895e550399c19bc956e95dafd728
+size 906004176

model-00017-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f4cf25bf54468c636e54fb7bf36fdf01ef92f64f583b8ad6f218b63861392ee
-size 906004192
+oid sha256:dd81a9f808fa3110b0c82a03a0127ea4b3a99fb93841883eeb222390297f20ee
+size 906004176

model-00018-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20bb6c28ec97f22f7145281bbf213fbe925c18ab1c2bfb64be4027b0572daa45
-size 906004192
+oid sha256:d60e2c9dc9b2c11560dcae80f447ce58242f1c97f5c5d020fd5e253157072b85
+size 906004176

model-00019-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8a4a831af510e8ebac56000b946e87e1635669906fad70b7a1731e0332528dc
-size 906004192
+oid sha256:a2fbccaae3c29cb15db20dd72e081ce3ff653e7e25fe406eb87fef54a1c94599
+size 906004176

model-00020-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:210097821d7e24dcf202964afe26fcfa738461dd4db152564b77dfc2400a6dae
-size 906004192
+oid sha256:6c02919de27efaf20c980d24abdfaf111eb7f383533185a704f29fad554def90
+size 906004176

model-00021-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25f75adba5416ffef022fc30e6d02cdcc1ce3165af75390b81ef31392f993475
-size 989923832
+oid sha256:87280f75c9383a25a4a2383d1e001c0c5f320f50f0f5fdf7af77637ba06303bf
+size 989923816

model-00022-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1bb2b30f271276fb6ee42af2f4b9117233a637fdec973c73a151c3a82096929
-size 906004192
+oid sha256:460470c9ccaf85b4acb9455b7346e7f2db1b1e6f261376f07a5a68f3c470389d
+size 906004176

model-00023-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba89c46e91a7b0eeca89e05e560a53a15592a0351c5183e90a59bd4be748fe15
-size 906004184
+oid sha256:2e26cec61dc09fbdb72e321f414d0ec3b4efb718e7fa7d628059a2b80345fd4f
+size 906004176

model-00024-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6241fda4d94ebf484c9b4aaab4ffaa5a636183aef5e9dd6c89cc0086801d546
-size 906004176
+oid sha256:fc5bc69e474536665c1317a1a50ae7e52fd54568c1aefe913abe673262e08550
+size 906004160

model-00025-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec8ec918b33179b29573c4f60594eefe2550ef42805d9f0a5c5987a1484a5e3e
-size 906004176
+oid sha256:bb3597bd7a01dcff6ef64c48f0ba8ffc2d23f402932bdc21ca03f349cdf1ad10
+size 906004160

model-00026-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8afb96f73769a37d9d3c7cdf8ff84e1c3ace7fcc7bbc526254c28d990a5ad718
-size 906004176
+oid sha256:13ff5cf63a526e38216d8478fe461aca13f216b16e6aa4593063eec44ae0ce1e
+size 906004160

model-00027-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7512d0c9ad00ad601ce5dcc666c7172c2c7e1753ae704f630e944deae0a5697
-size 989923824
+oid sha256:12efd2545a0007ff746b5108cafc336da952d38cf08e9688014b0a584477af0e
+size 989923808

model-00028-of-00028.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c809bfaf25a6bb8f3806c77c4bbefee0d716617e9a5d3859d115f4d57aae1e6
-size 788571816
+oid sha256:c95c2499a5d8a4e6ffc24557eb845a1ee9b5f0df35c75f53f96572ce46370077
+size 788571800