Hugging Face's logo

Tongjilibo
/
bert4torch_config

Model card Files Community
Tongjilibo commited on
Commit
f3c8733
•
1 Parent(s): 16f61c2

增加Qwen1.5

Browse files
Files changed (8) hide show
  1. Qwen1.5-0.5B-Chat/bert4torch_config.json +28 -0
  2. Qwen1.5-0.5B/bert4torch_config.json +28 -0
  3. Qwen1.5-1.8B-Chat/bert4torch_config.json +28 -0
  4. Qwen1.5-1.8B/bert4torch_config.json +28 -0
  5. Qwen1.5-14B-Chat/bert4torch_config.json +28 -0
  6. Qwen1.5-14B/bert4torch_config.json +28 -0
  7. Qwen1.5-7B-Chat/bert4torch_config.json +28 -0
  8. Qwen1.5-7B/bert4torch_config.json +28 -0
Qwen1.5-0.5B-Chat/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 2816,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 1024,
11
+ "num_attention_heads": 16,
12
+ "num_hidden_layers": 24,
13
+ "num_key_value_heads": 16,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": true,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 21,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151644, 151645],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-0.5B/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 2816,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 1024,
11
+ "num_attention_heads": 16,
12
+ "num_hidden_layers": 24,
13
+ "num_key_value_heads": 16,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": true,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 21,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151643],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-1.8B-Chat/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 5504,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 2048,
11
+ "num_attention_heads": 16,
12
+ "num_hidden_layers": 24,
13
+ "num_key_value_heads": 16,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 21,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151644, 151645],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-1.8B/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 5504,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 2048,
11
+ "num_attention_heads": 16,
12
+ "num_hidden_layers": 24,
13
+ "num_key_value_heads": 16,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 21,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151643],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-14B-Chat/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 13696,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 5120,
11
+ "num_attention_heads": 40,
12
+ "num_hidden_layers": 40,
13
+ "num_key_value_heads": 40,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 152064,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 35,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151644, 151645],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-14B/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 13696,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 5120,
11
+ "num_attention_heads": 40,
12
+ "num_hidden_layers": 40,
13
+ "num_key_value_heads": 40,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 152064,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 35,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151643],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-7B-Chat/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 11008,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 4096,
11
+ "num_attention_heads": 32,
12
+ "num_hidden_layers": 32,
13
+ "num_key_value_heads": 32,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 28,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151644, 151645],
27
+ "max_length": 32768}
28
+ }
Qwen1.5-7B/bert4torch_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "qwen2",
3
+ "hidden_act": "silu",
4
+ "attention_probs_dropout_prob": 0.0,
5
+ "bos_token_id": 151643,
6
+ "eos_token_id": 151645,
7
+ "intermediate_size": 11008,
8
+ "initializer_range": 0.02,
9
+ "layer_norm_eps": 1e-06,
10
+ "hidden_size": 4096,
11
+ "num_attention_heads": 32,
12
+ "num_hidden_layers": 32,
13
+ "num_key_value_heads": 32,
14
+ "rope_theta": 1000000.0,
15
+ "tie_word_embeddings": false,
16
+ "torch_dtype": "bfloat16",
17
+ "flash_attention": true,
18
+ "is_causal": true,
19
+ "vocab_size": 151936,
20
+ "segment_vocab_size": 0,
21
+ "skip_init": true,
22
+ "rope_rank": "updown",
23
+ "max_position_embeddings": 32768,
24
+ "sliding_window": 32768,
25
+ "max_window_layers": 28,
26
+ "generation_config": {"tokenizer_config": {"skip_special_tokens": true}, "end_id": [151643],
27
+ "max_length": 32768}
28
+ }