Dampfinchen commited on
Commit
1788223
1 Parent(s): 2261f2a

Fix query_pre_attn_scalar value

Browse files

See here https://huggingface.co/google/gemma-2-9b-it/commit/1937c70277fcc5f7fb0fc772fc5bc69378996e71

Files changed (1) hide show
  1. config.json +1 -1
config.json CHANGED
@@ -22,7 +22,7 @@
22
  "num_hidden_layers": 42,
23
  "num_key_value_heads": 8,
24
  "pad_token_id": 0,
25
- "query_pre_attn_scalar": 224,
26
  "rms_norm_eps": 1e-06,
27
  "rope_theta": 10000.0,
28
  "sliding_window": 4096,
 
22
  "num_hidden_layers": 42,
23
  "num_key_value_heads": 8,
24
  "pad_token_id": 0,
25
+ "query_pre_attn_scalar": 256,
26
  "rms_norm_eps": 1e-06,
27
  "rope_theta": 10000.0,
28
  "sliding_window": 4096,