| { | |
| "block_size": "8_8_8", | |
| "d_embed": 1024, | |
| "d_head": 64, | |
| "d_inner": 4096, | |
| "d_model": 1024, | |
| "dropact": 0.0, | |
| "dropatt": 0.1, | |
| "dropout": 0.1, | |
| "n_head": 16, | |
| "pool_q_only": true, | |
| "pooling_size": 2, | |
| "pooling_type": "mean", | |
| "separate_cls": true, | |
| "vocab_size": 50000 | |
| } |