scandinavian_solo

scandinavian_large.gin

@@ -10,7 +10,7 @@ import tasks
 
 MIXTURE_OR_TASK_NAME = "ncc_scandinavian_span_corruption_stream"
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
-TRAIN_STEPS = 1_700_000
+TRAIN_STEPS = 1_500_000
 DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
 INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_large/checkpoint_1500000"
 PjitPartitioner.num_partitions = 2

scandinavian_solo_large.gin

@@ -0,0 +1,18 @@
+include 't5x/examples/t5/mt5/large.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "ncc_scandinavian_span_corruption_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_700_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_large/checkpoint_1500000"
+PjitPartitioner.num_partitions = 2
+
+

train_large.sh

@@ -1,9 +1,9 @@
 PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
 T5X_DIR="../../t5x"  # directory where the t5x is cloned.
-MODEL_DIR="gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_large"
+MODEL_DIR="gs://nb-t5x-us-central2/scandinavian_t5x_large"
 export PYTHONPATH=${PROJECT_DIR}
 
 python3 ${T5X_DIR}/t5x/train.py \
   --gin_search_paths=${PROJECT_DIR} \
-  --gin_file="norwegian_large.gin" \
+  --gin_file="scandinavian_large.gin" \
   --gin.MODEL_DIR="'${MODEL_DIR}'" \