Hugging Face's logo

yifeihe3
/
phi_moe_embed

Text Generation
Transformers
Safetensors
mixtral
text-generation-inference
Inference Endpoints
Model card Files Community
yifeihe3 commited on
Commit
0020814
•
1 Parent(s): b753615

Upload MixtralForCausalLM

Browse files
Files changed (37) hide show
  1. config.json +36 -0
  2. generation_config.json +6 -0
  3. model-00001-of-00034.safetensors +3 -0
  4. model-00002-of-00034.safetensors +3 -0
  5. model-00003-of-00034.safetensors +3 -0
  6. model-00004-of-00034.safetensors +3 -0
  7. model-00005-of-00034.safetensors +3 -0
  8. model-00006-of-00034.safetensors +3 -0
  9. model-00007-of-00034.safetensors +3 -0
  10. model-00008-of-00034.safetensors +3 -0
  11. model-00009-of-00034.safetensors +3 -0
  12. model-00010-of-00034.safetensors +3 -0
  13. model-00011-of-00034.safetensors +3 -0
  14. model-00012-of-00034.safetensors +3 -0
  15. model-00013-of-00034.safetensors +3 -0
  16. model-00014-of-00034.safetensors +3 -0
  17. model-00015-of-00034.safetensors +3 -0
  18. model-00016-of-00034.safetensors +3 -0
  19. model-00017-of-00034.safetensors +3 -0
  20. model-00018-of-00034.safetensors +3 -0
  21. model-00019-of-00034.safetensors +3 -0
  22. model-00020-of-00034.safetensors +3 -0
  23. model-00021-of-00034.safetensors +3 -0
  24. model-00022-of-00034.safetensors +3 -0
  25. model-00023-of-00034.safetensors +3 -0
  26. model-00024-of-00034.safetensors +3 -0
  27. model-00025-of-00034.safetensors +3 -0
  28. model-00026-of-00034.safetensors +3 -0
  29. model-00027-of-00034.safetensors +3 -0
  30. model-00028-of-00034.safetensors +3 -0
  31. model-00029-of-00034.safetensors +3 -0
  32. model-00030-of-00034.safetensors +3 -0
  33. model-00031-of-00034.safetensors +3 -0
  34. model-00032-of-00034.safetensors +3 -0
  35. model-00033-of-00034.safetensors +3 -0
  36. model-00034-of-00034.safetensors +3 -0
  37. model.safetensors.index.json +0 -0
config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MixtralForCausalLM"
4
+ ],
5
+ "attention_bias": true,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "input_jitter_noise": 0.01,
13
+ "intermediate_size": 6400,
14
+ "lm_head_bias": true,
15
+ "lora_rank": 32,
16
+ "max_position_embeddings": 4096,
17
+ "model_type": "mixtral",
18
+ "num_attention_heads": 32,
19
+ "num_experts_per_tok": 2,
20
+ "num_hidden_layers": 32,
21
+ "num_key_value_heads": 8,
22
+ "num_local_experts": 16,
23
+ "output_router_logits": false,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_theta": 10000.0,
26
+ "router_aux_loss_coef": 0.0,
27
+ "router_jitter_noise": 0.01,
28
+ "separate_lora": true,
29
+ "sliding_window": 2047,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.45.2",
33
+ "use_cache": true,
34
+ "use_lora": true,
35
+ "vocab_size": 32064
36
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.45.2"
6
+ }
model-00001-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:468772608af3879f3582a629d66e5ffb8fdb39f19c56c3aeed3a440d28dd6a4d
3
+ size 4992842152
model-00002-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62c1d67cd24db8316e00621e8ea0f4ea69f043437c59211df786e3df6dd01ab4
3
+ size 4991860096
model-00003-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cddc0740f668ea30eb42f24a3188d9712e8f716f706335de08c50e54ef712f7a
3
+ size 4991860096
model-00004-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99fc25881faa05f7a8afb6d0460d18b9faef646fd89591082e5d8dedac026597
3
+ size 4991860104
model-00005-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:290ea4b1b746b8fc80f6121435b5571ec2ec444179545d2acd6ab346d0be8fde
3
+ size 4991860104
model-00006-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91c81754f29ea15044bfd7ae6df6d7c6fc320eeca6b5ce1dccee48690bbb6a66
3
+ size 4991860104
model-00007-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e122d346b96654576c2aa850fe37bea4afe65cad7e773838093186fbd02efca7
3
+ size 4991860104
model-00008-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5303262321c482e9728a29947b449557195338a68170f4b67114205ef1fb664d
3
+ size 4991860104
model-00009-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:575b005a1511317c4f01f677e6cf9c100c80b9a759516b9de52a476baaa5e69a
3
+ size 4991860104
model-00010-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63030d36ac99f214d2075c92f05c79ef5d6310c8bc28329e0a01665e25fa35b4
3
+ size 4991860104
model-00011-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98eed1afa446c7d0b23fb943206ef29a529ba4fd7e9ffe0b32be5ec2eb5d15fd
3
+ size 4991860128
model-00012-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12e1c3325246dc4b87195d7f795e7cdb78e0febf2abe9a83b4bb7e9ddd66234b
3
+ size 4991860168
model-00013-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b04d7071d8a4c0179164563c9465b5bf192ec1938658db916e424dc2259985fc
3
+ size 4991860168
model-00014-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3de9b1bedb558e5696a80864f8a14fcb44fe3bad9ff478dad3b905f980e5279c
3
+ size 4991860168
model-00015-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c1f4e7f59bfe41bced6fe89c1f80699813e9155a533065e4d6d496c86458368
3
+ size 4991860168
model-00016-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9694f2fc0732facade721104bfb70a44a8088612bbe99b9052aeffdbd5af2034
3
+ size 4991860168
model-00017-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e71529c61ba4bd55b43d3e5aaa50a9d5d79a14aed2f59c9afb9aab6ef2f3456
3
+ size 4991860168
model-00018-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a57ffdb23b853e7428726e0f1db0de964fb6c2f1068c8a3886722239da7e982
3
+ size 4991860168
model-00019-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe07e06a83b147a74f45b88b98c1f8d5a925f365f3047d8650deef5d089309bc
3
+ size 4991860168
model-00020-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42931ba9fa4c088497e0cb4b6530b1f0f684962cf5b164932e8f708d8957b17c
3
+ size 4991860168
model-00021-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d478e0bac8dc806125ecb427c15b257aef65d2fc5ad31669ebe68f4752ca1677
3
+ size 4991860168
model-00022-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79e2c30c6e23ca46283f693a206a4a21a9365881394265ec8d2bbc85d15304b2
3
+ size 4995504816
model-00023-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f12656064f1d8920f6cac93350ffdaf1176a01f67e5a782772aa8421178f962e
3
+ size 4924668688
model-00024-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aec119552c920a2a006c697cf6f0baa109c68bc3d7a5260e8f2ae7c4759ebc43
3
+ size 4991860152
model-00025-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f6a418044e61bcde5e1ca8e62183e4f2e0cdbe8b8dfde01ed841acffa62d08c
3
+ size 4991860152
model-00026-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17400df5ab0f57ca5890787d79b9d413c64cd7faaea5da7dc2a61d8dda027ea1
3
+ size 4991860152
model-00027-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f68ba5802d736fce3f293304351d9a926bcdf058c618a7a217843e078c8687b
3
+ size 4991860152
model-00028-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b660d0a958dcec6a072e42a297eba8a2b58deb28763f00bfd3281088959b090
3
+ size 4991860168
model-00029-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5360e244245125c95b5070b9eab9ca864e6929b0abe2232f019bc6d7fefdc668
3
+ size 4991860168
model-00030-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ebb03488fda8ab7786d818177f392db4090f75d8b80de8336458700728e63d7
3
+ size 4991860168
model-00031-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3bc2b01a9a7bb9b315ae764a883b3cf376c325a3951cbcd9b08317d4c0e785c4
3
+ size 4991860168
model-00032-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b9da40e2fbf838852c7eb3c92f0854b545a9e525cfdbcbf5d550cd20c02e18d
3
+ size 4991860168
model-00033-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be4e396abaabb3921112c208face5807e2473d2561e4546b3c2391e52b55df0c
3
+ size 4991860168
model-00034-of-00034.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b922175e93b111314e7e37559f9900d8fa977ab01884bf149eaf631e17adef8
3
+ size 2832434048
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff