Hugging Face's logo

ibivibiv
/
orthorus-125b-moe

Text Generation
Transformers
Safetensors
English
mixtral
Mixture of Experts
Eval Results
text-generation-inference
Inference Endpoints
Model card Files Community
1
ibivibiv commited on
Commit
1a742de
•
1 Parent(s): 9bad321

Upload MixtralForCausalLM

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +35 -0
  2. generation_config.json +8 -0
  3. model-00001-of-00108.safetensors +3 -0
  4. model-00002-of-00108.safetensors +3 -0
  5. model-00003-of-00108.safetensors +3 -0
  6. model-00004-of-00108.safetensors +3 -0
  7. model-00005-of-00108.safetensors +3 -0
  8. model-00006-of-00108.safetensors +3 -0
  9. model-00007-of-00108.safetensors +3 -0
  10. model-00008-of-00108.safetensors +3 -0
  11. model-00009-of-00108.safetensors +3 -0
  12. model-00010-of-00108.safetensors +3 -0
  13. model-00011-of-00108.safetensors +3 -0
  14. model-00012-of-00108.safetensors +3 -0
  15. model-00013-of-00108.safetensors +3 -0
  16. model-00014-of-00108.safetensors +3 -0
  17. model-00015-of-00108.safetensors +3 -0
  18. model-00016-of-00108.safetensors +3 -0
  19. model-00017-of-00108.safetensors +3 -0
  20. model-00018-of-00108.safetensors +3 -0
  21. model-00019-of-00108.safetensors +3 -0
  22. model-00020-of-00108.safetensors +3 -0
  23. model-00021-of-00108.safetensors +3 -0
  24. model-00022-of-00108.safetensors +3 -0
  25. model-00023-of-00108.safetensors +3 -0
  26. model-00024-of-00108.safetensors +3 -0
  27. model-00025-of-00108.safetensors +3 -0
  28. model-00026-of-00108.safetensors +3 -0
  29. model-00027-of-00108.safetensors +3 -0
  30. model-00028-of-00108.safetensors +3 -0
  31. model-00029-of-00108.safetensors +3 -0
  32. model-00030-of-00108.safetensors +3 -0
  33. model-00031-of-00108.safetensors +3 -0
  34. model-00032-of-00108.safetensors +3 -0
  35. model-00033-of-00108.safetensors +3 -0
  36. model-00034-of-00108.safetensors +3 -0
  37. model-00035-of-00108.safetensors +3 -0
  38. model-00036-of-00108.safetensors +3 -0
  39. model-00037-of-00108.safetensors +3 -0
  40. model-00038-of-00108.safetensors +3 -0
  41. model-00039-of-00108.safetensors +3 -0
  42. model-00040-of-00108.safetensors +3 -0
  43. model-00041-of-00108.safetensors +3 -0
  44. model-00042-of-00108.safetensors +3 -0
  45. model-00043-of-00108.safetensors +3 -0
  46. model-00044-of-00108.safetensors +3 -0
  47. model-00045-of-00108.safetensors +3 -0
  48. model-00046-of-00108.safetensors +3 -0
  49. model-00047-of-00108.safetensors +3 -0
  50. model-00048-of-00108.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./superbeyonder",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 8192,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 28672,
14
+ "max_length": 4096,
15
+ "max_position_embeddings": 2048,
16
+ "model_type": "mixtral",
17
+ "num_attention_heads": 64,
18
+ "num_experts_per_tok": 2,
19
+ "num_hidden_layers": 80,
20
+ "num_key_value_heads": 8,
21
+ "num_local_experts": 2,
22
+ "output_router_logits": false,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-05,
26
+ "rope_scaling": null,
27
+ "rope_theta": 10000.0,
28
+ "router_aux_loss_coef": 0.001,
29
+ "sliding_window": null,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.36.2",
33
+ "use_cache": true,
34
+ "vocab_size": 32000
35
+ }
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "max_length": 4096,
6
+ "pad_token_id": 0,
7
+ "transformers_version": "4.36.2"
8
+ }
model-00001-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2f824bb64db79c44c6ee3779001c5dc9e6dc55b5e38d4553200855b415c99fc
3
+ size 4471194712
model-00002-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54d6e5f04cfdf4e6b9fbc1d8cb55f5bd236caa17d04f599ed539bbd788467645
3
+ size 4362208592
model-00003-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b66e27fb226ec679a0032837be16bd7a5bdaa28797990f4228f5b814da396da
3
+ size 4999677048
model-00004-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b148aa43369f9a9dbc21a541f344192a9ff408a53067f5261e28aa46e8db7f5d
3
+ size 4999676936
model-00005-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f36a73ec196ac36c5a4d0adc22a8acde8f6ed564f2331baf4a2b3a5c6103f399
3
+ size 4362208592
model-00006-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b22a89a1442b632655db36cbcdf26b9837350ab889043cf2ee59bcc253f19aa
3
+ size 4362208592
model-00007-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d63d4fffcbde7a58553b59e61e6e28fe76effe14d2bc31c2b6d3f995da3d4d1
3
+ size 4999677048
model-00008-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f79c9f8a6f6ac527819f2c0fff704e0612501b95bbbf4b20debd6defce94820d
3
+ size 4999676936
model-00009-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b783f77a641b1128b1efc503c6b83bb27761f19c1a8b0661232218a3958fb369
3
+ size 4362208592
model-00010-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dab22c10227c6cf866e62af05104148b828a1e0035d4f352523ed134afd0a4a8
3
+ size 4362208592
model-00011-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af8dd63f7beb6b269620e1f01f392f5f867fee1b48d2dccbc28f364d5567a1b9
3
+ size 4999677048
model-00012-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef19ebdd4a1ca0e929c90c0bcc0ee8ffa9dce79709fa5c3f4897cca2f441908d
3
+ size 4999676936
model-00013-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9de47a5cc712db7ada962e97e46efae918e49ece332cdaaa67ff644f4d02c6f9
3
+ size 4362208592
model-00014-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:865af2acaf773da401f01de675f8aa92a563d92be966c690ae61e01ddae2e927
3
+ size 4362208600
model-00015-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfaa7902c960646dbd72f312db796c2b5e620f75d0568bbd5b2efc96d6c1d5b6
3
+ size 4999677056
model-00016-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:910a7de4b11d0418bf34cdf363f054b33ac33984b15062940a64d8daf069bf6f
3
+ size 4999676944
model-00017-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c0b4ec25d85e719117aa193d9a7ee77d82adcf15717538afb00dc8886277891
3
+ size 4362208600
model-00018-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50a44463d600dced8ac95ebe465238b81c69a790f3a218693e53cb4dd93e8fe9
3
+ size 4362208608
model-00019-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03f1914b0e639222792e12e98b0096bbe58596497a11ee4c4cacfb9a500aabb0
3
+ size 4999677056
model-00020-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62adbb7e1235c2c8f2e8e60efe7448fbae5cc97dfd93520ca3a738cbaa368c36
3
+ size 4999676944
model-00021-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aefbdca5681a2d12e2e843013d30baf432622b1034e4560c5581aaf0f89ac77d
3
+ size 4362208600
model-00022-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b5957b1c15b08e8b9778305167d7f766aed280b889e236a55d8f71371d20bb2
3
+ size 4362208608
model-00023-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aea10e16ccdf0dc1adbcd669291549231bf579abd2afc390b027d0750a364156
3
+ size 4999677056
model-00024-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3d5a24c0476050ffb7cd6248c8afed5548380d6c2ce610007fa7c821d2ed806
3
+ size 4999676944
model-00025-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b15196fde88434bba20f68b289ffdd2d23a9b9488b67881d076af60db6742c0b
3
+ size 4362208600
model-00026-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e77258c981ca81688cca2a3cfea0a2f685b43bade1d7faffaf5904840c9c57f1
3
+ size 4362208608
model-00027-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f78387a16253e86b0a68ec27717185b7644c515a6de4f58f5c3d11aa155df80
3
+ size 4999677056
model-00028-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:653ae7f35624ae37ec823b22aeac85ecbbab7ff3699dbc4ac0b166b528fb38d6
3
+ size 4999676944
model-00029-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a558525d6bf0fa89b5170d4b07965e5f66752e8d74de482d6845d94c0ccc59c7
3
+ size 4362208600
model-00030-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac005fd0c955eb607f137360fe5f35524083386375d35f44d7938c86fd3a3104
3
+ size 4362208608
model-00031-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f0ab44c5b31ef54a2f86ea567153c03c255dc36855f1037887d58c2a7bcbb07
3
+ size 4999677056
model-00032-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c558eefa73514903466f8a4482de7a14d63828e93f11f386b60fc6e0a9ed160
3
+ size 4999676944
model-00033-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbdbda3b6d6db73b5727cb2cee47955ec8a40c9093218e3f0b3933bc0dcbfe33
3
+ size 4362208600
model-00034-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c92b5e960bef70bd682229a1d27f221e040201941ecdedefb2351f08a4db486c
3
+ size 4362208608
model-00035-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:302381c7554795967f5687ee5425e7d878c41ab08460f52e34f86d8d59bca165
3
+ size 4999677056
model-00036-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a197c32cbc987115a2fdf7c29106394130077dfa2bdddd19bcb3d1cea855d6d
3
+ size 4999676944
model-00037-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b39f4401c9f9b85d573046f66f67c2ab408cc06aff7fe5588c9c356f89c837f6
3
+ size 4362208600
model-00038-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36b8b69b734f5d720cb92945460aa3ea1514d798b5effbd2ae773aa9dfd5b8f8
3
+ size 4362208608
model-00039-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96296c65fe4397b6003209e223b0dc0272fcb4c3c22442116ff688127094da39
3
+ size 4999677056
model-00040-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d018fcc4b0c44d76cf42b81d77fb0a09da74377821324f6e66010adf878ea68
3
+ size 4999676944
model-00041-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:545d84a4527e8f94a21c0f58760140d4596f6c0d0df405429b87594f2cb52e71
3
+ size 4362208600
model-00042-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25bf8408bad815366cbf9422df9f4c68d3445f3390e8942bb479851f6f8dec46
3
+ size 4362208608
model-00043-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d80566348bde16d627937ee7dd0d330273b8d9a1b66060302f1d00cea592175
3
+ size 4999677056
model-00044-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:080584e45195a99c6343fe832712595667cdcfa9c6065c2f89cccd3d406cf22a
3
+ size 4999676944
model-00045-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:528228bab9247fdd5114b895f8fbdce5deed6737b2dc416ead16b378952d9b7d
3
+ size 4362208600
model-00046-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ed8e73d6b66107b5b528cd769837043b696aa8cbf7bb3f22fe747b264dabec5
3
+ size 4362208608
model-00047-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b21d23ba8abd05afb246bfe31a13e272f612619b992c6ae47c99093f65889ecf
3
+ size 4999677056
model-00048-of-00108.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e293a2e232a879938dea305c09acc53718dbdd0b73e4d3bc574220bbaaf044ef
3
+ size 4999676944