include 't5x/examples/t5/mt5/xl.gin' include 'pretrain_cont.gin' #include 't5x/configs/runs/pretrain.gin' #iinclude 't5x/configs/runs/finetune.gin' # Register necessary SeqIO Tasks/Mixtures. import t5.data.mixtures import tasks MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_prefix_lm_stream" TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512} TRAIN_STEPS = 1_800_000 DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this. INITIAL_CHECKPOINT_PATH = "gs://north-t5x/pretrained_models/xl/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_xl/checkpoint_1700000" PjitPartitioner.num_partitions = 4 utils.SaveCheckpointConfig.period = 5000 utils.SaveCheckpointConfig.keep = 3