config.json

{
 "dim": 320,
 "n_layers": 12,
 "n_heads": 20,
 "vocab_size": 2362,
 "dim_ffn": 960,
 "norm_eps": 1e-05,
 "multiple_of": 64,
 "padding_idx": 2,
 "max_seq_len": 512,
 "initializer_range": 0.02
}