juancopi81's picture
Add t5x and mt3 models
b100e1c
raw
history blame
368 Bytes
# T5.1.1 Small model.
include 't5x/examples/t5/t5_1_0/base.gin' # imports vocab, optimizer and model.
# ------------------- Network specification overrides --------------------------
network.Transformer.config = @network.T5Config()
network.T5Config:
emb_dim = 512
num_heads = 8
num_encoder_layers = 6
num_decoder_layers = 6
head_dim = 64
mlp_dim = 2048