bias: true
block_size: 2048
gelu_approximate: none
head_size: 64
hf_config:
  name: pythia-160m
  org: EleutherAI
intermediate_size: 3072
lm_head_bias: false
mlp_class_name: GptNeoxMLP
n_embd: 768
n_expert: 0
n_expert_per_token: 0
n_head: 12
n_layer: 12
n_query_groups: 12
name: pythia-160m
norm_class_name: LayerNorm
norm_eps: 1.0e-05
padded_vocab_size: 50304
padding_multiple: 128
parallel_residual: true
rope_base: 10000
rope_condense_ratio: 1
rotary_percentage: 0.25
scale_embeddings: false
shared_attention_norm: false
vocab_size: 50254