Upload folder using huggingface_hub

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "NeverSleep/NoromaidxOpenGPT4-2",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+slices:
+  - sources:
+      - model: Fredithefish/MysticRP4
+        layer_range: [0, 32]
+      - model: NeverSleep/NoromaidxOpenGPT4-2
+        layer_range: [0, 32]
+
+merge_method: slerp
+base_model: NeverSleep/NoromaidxOpenGPT4-2
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5 # fallback for rest of tensors
+dtype: bfloat16

model-00001-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65c9b93324177d91637b888fc9cc57d43195e3dd9e901af5f1af8eac2830dda9
+size 1963199472

model-00002-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e49fb6d1f87ab9d56a4c411205f3a391ab9186fc47249db67faf528a16bee00
+size 1996556608

model-00003-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d12a94ca1a2a66c8c6ac3cc8d5f059366794e0d2080c42092b24f62acc448d8
+size 1956793496

model-00004-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c08893c48f7947c4ec9ccbec768069ca60f26088dcad297e5be01f9a80f256
+size 1996490952

model-00005-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aaeefd5051251815423cddc2c03877f28f066d353dfca3889827e1823cf7d3a
+size 1996524632

model-00006-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b0f8494e0f9c477714d23ed168e99517f8bc377708c3736cd479e042682c392
+size 1979853920

model-00007-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f9af2a461670992cb4795bd3a617f94f909dc330883ad6f968ac2371c054ff6
+size 1912613160

model-00008-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f56861120939a733a22203128d170145f09c252a20580dd4277fbe54ab5e62b
+size 1996507568

model-00009-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17fd575bf091b30f7f6ef107126b1757100b15406334ee898593a750e531024e
+size 1963019112

model-00010-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cd00ccef65b98789b51f0d246be570fda394ca5b9da366c4761277fce6b606b
+size 1996490952

model-00011-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8586ece5f1b101310faa3623d692e1d9fcee1d72fc49392a0ca98c6ea7966a3
+size 1963068152

model-00012-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd5045976ae6afa2733ef2cef6eda93fd341f559bb7f257361a29178f38391a
+size 1996490952

model-00013-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5608c17da0d1a93e6468ed0c4a95f8264a0ec95b4bb153452eef190a89e7cba
+size 1929489960

model-00014-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4111873aec39a62bc9e7e21f38ad8e1ed63dd43b7de0866e5338df0674d7a0e
+size 1996556592

model-00015-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eecb758db2f89bbdaeb37e962a6ab8d23e961632b4f853eb70c07619c4746ee2
+size 1962945160

model-00016-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:039ac84c74253629e88fd60f6e89ceac3cc811db09186ea640b07902aa776637
+size 1996573240

model-00017-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5975ae9518428fa7e3874765cd609fff68f5fc7336252c6efc1412569d18a64b
+size 1962936872

model-00018-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f29f0dfccc1c3925eeee909b904882717cea9f22a50f5f5d2679f1f3cf9ac43
+size 1996507584

model-00019-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53feefcbf022d6c56ad016ac1d28f7c7b00610dec072d4f87bde76ef3fbbbeee
+size 1996631048

model-00020-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ccb6e7030dc96fc392d5cfcc6cd17ddc7bc105676db695f9a99acd52a96260a
+size 1979722632

model-00021-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:625216d6987257d081c3a90a426a551366d1d7d73467546e2e3f81cb6ef6dc24
+size 1996655968

model-00022-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec51ed86a70132105906c5d2fae032dfd536c6b3cbf9630e9d55b0b62d1baff
+size 1996490968

model-00023-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c6416bf8f8cf0c44e000f6920180ed811bf6431efa6019a8d7b6b950509dbc8
+size 1962953488

model-00024-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e389d21bb5b4db29f750e315aec967d3e545ff933cdb291e4457504d2bcc0bf8
+size 1996556624

model-00025-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4477ba071b66e140899b2b4729a63cbc8127ca16c6c12b2aa586e79006bd0ec9
+size 1996490968

model-00026-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a983f590e3cbe0748ab4d211210c0aa3e8b51bc0f38232de9f3827ac91cdb42
+size 1963084800

model-00027-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9a4d1cbad9b203ee4ab5b375b7807c59670c9c4786319e16ac9ccb6d5e4c728
+size 1996490968

model-00028-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2b8946e40b41b7bf563d42e0ee9f6f717a6988b7ca772a78ba13e68eb4a1839
+size 1929416016

model-00029-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b1f45181e2b6897299fdf33554c1884c22024107906a49232f8f941db4ae943
+size 1996639352

model-00030-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fc284b6b4562cb9976fabe3c761d38bdc7ee4a35d245ea5d11a458b63b054c
+size 1929382328

model-00031-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1633805d488f8463ee4161676097589f063b180051d8bb37620e54617bffb54a
+size 1963101416

model-00032-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3183dc280c418ae818e39a5d4ba06a06d2052d970e3147c6617793d8b006f1c8
+size 1996490968

model-00033-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1140dbc1e7ee74a78ca3ccb2dc498beaef26a39694b43514ed26ded90b040db4
+size 1962953488

model-00034-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502253e5f3a914dd057da129b0af101ed47a4f9f40bac4d938f895af2dffc4f1
+size 1996556624

model-00035-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea996827b14501890171b1c5c082241b38655eecd7259fdc2aba0b8d7ac7cc5a
+size 1962936872

model-00036-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6d0c308b6b673b30a9351d8f5827991bd981f19f4de80691d1f20508fc8234
+size 1996638888

model-00037-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a7a164be9b487af2dfafdb5be03500cfa7e23876a88deaa70150648da1d5637
+size 1962936872

model-00038-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12e8e23ad296cbd8a17173668cc3bea91c0ccf240ee9a59fe92422b9c4112fc3
+size 1996598624

model-00039-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46651fbd409a81e07c6fb62cc11814661cfc0145cb8f50bf77cba9db35679d62
+size 1929390632

model-00040-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa0c6c0d0bf7f3f01dbea5171fe180a1222848628972823b35058c8323f35ee2
+size 1996490968

model-00041-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a482791695f5ce51cae641667c93ddab672aee85e7f49fa2c82ba1441d7f23
+size 1962953488

model-00042-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d78f35fae9799eae9abdefe3dbe5133fc9e9f3f7dbb2ccde3fe09a9fb6f13f7
+size 1996490968

model-00043-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16145ae71babf9a52be8cbaaf57597806a444bffc9c71643af4cf2bdcf428de5
+size 1962953488

model-00044-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73d88bcb0f4d7ab1f7b12221093bae855d190f68b1d6d9a73b6b4a325cbbe964
+size 1996556624

model-00045-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51caa2812124cdb34af6c53a24ed26914609e11bad94a286f8ce0f1178bbfa5d
+size 1879050320

model-00046-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fefe7ffa8d067b85b665b695bfa7c12a17b207c825d0b5ebc41163fe3747e1a0
+size 1990315152

model-00047-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe77e30da668454457c1746df9801332be1a1253a705a2b56723f0f5fc4e13e2
+size 1962961792

model-00048-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68780731751cb3d23b8e57539589445e729190cef59ebb11e31f308f11d49cb
+size 587203248