| { |
| "sequence_len": 8192, |
| "vocab_size": 131072, |
| "n_layer": 16, |
| "n_head": 8, |
| "n_kv_head": 8, |
| "n_embd": 1024, |
| "moe_num_experts": 8, |
| "moe_top_k": 2, |
| "moe_layer_interval": 3, |
| "moe_group_size": 4, |
| "moe_expert_intermediate_size": 1792, |
| "moe_adjugate_intermediate_size": 0, |
| "moe_adjugate_scale": 0.05, |
| "moe_router_aux_loss_coef": 0.015, |
| "moe_router_bias_lr": 0.001, |
| "moe_activation_checkpoint": true, |
| "moe_capacity_factor": 0.75, |
| "rotary_scaling_type": "yarn", |
| "rotary_scale_factor": 4.0, |
| "residual_scale": -1.0, |
| "attn_dropout": 0.01, |
| "label_smoothing": 0.0, |
| "z_loss_weight": 0.0, |
| "use_flash_attention": true, |
| "domain_router_dim": 32, |
| "num_domain_tags": 128, |
| "domain_router_features": { |
| "dataset": { |
| "capacity": 128, |
| "mode": "one_hot" |
| }, |
| "quality": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "specialty": { |
| "capacity": 64, |
| "mode": "one_hot" |
| }, |
| "modality": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "language": { |
| "capacity": 32, |
| "mode": "one_hot" |
| }, |
| "origin": { |
| "capacity": 8, |
| "mode": "one_hot" |
| } |
| } |
| } |
