File size: 685 Bytes
c1daee1 d3f25d9 481169b c1daee1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
include 't5x/examples/t5/mt5/large.gin'
include 'pretrain_cont.gin'
#include 't5x/configs/runs/pretrain.gin'
#iinclude 't5x/configs/runs/finetune.gin'
# Register necessary SeqIO Tasks/Mixtures.
import t5.data.mixtures
import tasks
MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_span_corruption_stream"
TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
TRAIN_STEPS = 1_700_000
DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_large/checkpoint_1500000"
PjitPartitioner.num_partitions = 2
utils.SaveCheckpointConfig.period = 5000
utils.SaveCheckpointConfig.keep = 3
|