onkar.pandit commited on
Commit
9cec009
1 Parent(s): a9a0df8

config update

Browse files
Files changed (1) hide show
  1. config.json +42 -32
config.json CHANGED
@@ -1,33 +1,43 @@
1
  {
2
- "attn_pdrop": 0.0,
3
- "scale_attn_weights": true,
4
- "resid_pdrop": 0.0,
5
- "mup_embeddings_scale": 14.6,
6
- "n_inner": 19114,
7
- "n_embd": 7168,
8
- "layer_norm_epsilon": 1e-05,
9
- "n_positions": 8192,
10
- "activation_function": "swiglu",
11
- "n_head": 56,
12
- "n_layer": 48,
13
- "mup_output_alpha": 2.22,
14
- "mup_width_scale": 0.03571428571428571,
15
- "position_embedding_type": "alibi",
16
- "mup_scale_qk_dot_by_d": true,
17
- "tie_word_embeddings": true,
18
- "vocab_size": 84992,
19
- "embd_pdrop": 0.0,
20
- "model_type": "jais",
21
- "use_cache": true,
22
- "auto_map": {
23
- "AutoConfig": "configuration_jais.JAISConfig",
24
- "AutoModel": "modeling_jais.JAISModel",
25
- "AutoModelForCausalLM": "modeling_jais.JAISLMHeadModel",
26
- "AutoModelForQuestionAnswering": "modeling_jais.JAISForQuestionAnswering",
27
- "AutoModelForSequenceClassification": "modeling_jais.JAISForSequenceClassification",
28
- "AutoModelForTokenClassification": "modeling_jais.JAISForTokenClassification"
29
- },
30
- "eos_token_id": 0,
31
- "pad_token_id": 0,
32
- "bos_token_id": 0
33
- }
 
 
 
 
 
 
 
 
 
 
 
1
  {
2
+ "_name_or_path": "core42/jais-30b-chat-v3",
3
+ "activation_function": "swiglu",
4
+ "alibi_scaling": null,
5
+ "architectures": [
6
+ "JAISLMHeadModel"
7
+ ],
8
+ "attn_pdrop": 0.0,
9
+ "auto_map": {
10
+ "AutoConfig": "configuration_jais.JAISConfig",
11
+ "AutoModel": "modeling_jais.JAISModel",
12
+ "AutoModelForCausalLM": "modeling_jais.JAISLMHeadModel",
13
+ "AutoModelForQuestionAnswering": "modeling_jais.JAISForQuestionAnswering",
14
+ "AutoModelForSequenceClassification": "modeling_jais.JAISForSequenceClassification",
15
+ "AutoModelForTokenClassification": "modeling_jais.JAISForTokenClassification"
16
+ },
17
+ "bos_token_id": 0,
18
+ "embd_pdrop": 0.0,
19
+ "eos_token_id": 0,
20
+ "initializer_range": 0.02,
21
+ "layer_norm_epsilon": 1e-05,
22
+ "model_type": "jais",
23
+ "mup_embeddings_scale": 14.6,
24
+ "mup_output_alpha": 2.22,
25
+ "mup_scale_qk_dot_by_d": true,
26
+ "mup_width_scale": 0.03571428571428571,
27
+ "n_embd": 7168,
28
+ "n_head": 56,
29
+ "n_inner": 19114,
30
+ "n_layer": 48,
31
+ "n_positions": 8192,
32
+ "pad_token_id": 0,
33
+ "position_embedding_type": "alibi",
34
+ "reorder_and_upcast_attn": false,
35
+ "resid_pdrop": 0.0,
36
+ "scale_attn_by_inverse_layer_idx": false,
37
+ "scale_attn_weights": true,
38
+ "tie_word_embeddings": true,
39
+ "torch_dtype": "float32",
40
+ "transformers_version": "4.36.2",
41
+ "use_cache": true,
42
+ "vocab_size": 84992
43
+ }