zpn commited on
Commit
11c8ebe
1 Parent(s): 1d2e130

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +2 -2
config.json CHANGED
@@ -24,7 +24,7 @@
24
  "n_head": 12,
25
  "n_inner": 3072,
26
  "n_layer": 12,
27
- "n_positions": 2048,
28
  "pad_vocab_size_multiple": 64,
29
  "parallel_block": false,
30
  "parallel_block_tied_norm": false,
@@ -36,7 +36,7 @@
36
  "rotary_emb_fraction": 1.0,
37
  "rotary_emb_interleaved": false,
38
  "rotary_emb_scale_base": null,
39
- "rotary_scaling_factor": null,
40
  "scale_attn_by_inverse_layer_idx": false,
41
  "scale_attn_weights": true,
42
  "summary_activation": null,
 
24
  "n_head": 12,
25
  "n_inner": 3072,
26
  "n_layer": 12,
27
+ "n_positions": 8192,
28
  "pad_vocab_size_multiple": 64,
29
  "parallel_block": false,
30
  "parallel_block_tied_norm": false,
 
36
  "rotary_emb_fraction": 1.0,
37
  "rotary_emb_interleaved": false,
38
  "rotary_emb_scale_base": null,
39
+ "rotary_scaling_factor": 2,
40
  "scale_attn_by_inverse_layer_idx": false,
41
  "scale_attn_weights": true,
42
  "summary_activation": null,