File size: 731 Bytes
c262fb6
f446315
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# T5.1.1 Efficient large nl36 model.

import seqio
include 't5x/examples/t5/t5_1_1/base.gin'  # imports vocab, optimizer and model.

# ------------------- Network specification overrides --------------------------
network.Transformer.config = @network.T5Config()
network.T5Config:
  emb_dim = 1024
  num_heads = 16
  num_encoder_layers = 36
  num_decoder_layers = 36
  head_dim = 64
  mlp_dim = 4096

# ------------------- Model specification overrides --------------------------
VOCABULARY = @seqio.SentencePieceVocabulary()
seqio.SentencePieceVocabulary.sentencepiece_model_file = "spiece.model"

MODEL = @models.EncoderDecoderModel()
models.EncoderDecoderModel:
  input_vocabulary = %VOCABULARY
  output_vocabulary = %VOCABULARY