| include 't5x/examples/t5/mt5/large.gin' | |
| include 'pretrain_cont.gin' | |
| #include 't5x/configs/runs/pretrain.gin' | |
| #iinclude 't5x/configs/runs/finetune.gin' | |
| # Register necessary SeqIO Tasks/Mixtures. | |
| import t5.data.mixtures | |
| import tasks | |
| MIXTURE_OR_TASK_NAME = "ncc_english_span_corruption_stream" | |
| TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512} | |
| TRAIN_STEPS = 1_500_000 | |
| DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this. | |
| INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000" | |
| PjitPartitioner.num_partitions = 1 | |