lm models for nynorsk

balanced_bokmaal_nynorsk_lm_base.gin

@@ -0,0 +1,21 @@
+include 't5x/examples/t5/mt5/base.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_lm_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_800_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://north-t5x/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_base/checkpoint_1700000"
+#PjitPartitioner.num_partitions = 2
+utils.SaveCheckpointConfig.period = 5000
+utils.SaveCheckpointConfig.keep = 3
+
+
+

balanced_bokmaal_nynorsk_lm_large.gin

@@ -0,0 +1,21 @@
+include 't5x/examples/t5/mt5/large.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_lm_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_800_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://north-t5x/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_large/checkpoint_1700000"
+#PjitPartitioner.num_partitions = 2
+utils.SaveCheckpointConfig.period = 5000
+utils.SaveCheckpointConfig.keep = 3
+
+
+

balanced_bokmaal_nynorsk_lm_small.gin

@@ -0,0 +1,21 @@
+include 't5x/examples/t5/mt5/small.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_lm_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_800_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://north-t5x/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_small/checkpoint_1700000"
+#PjitPartitioner.num_partitions = 2
+utils.SaveCheckpointConfig.period = 5000
+utils.SaveCheckpointConfig.keep = 3
+
+
+

balanced_bokmaal_nynorsk_lm_xl.gin

@@ -0,0 +1,21 @@
+include 't5x/examples/t5/mt5/xl.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_lm_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_800_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://north-t5x/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_xl/checkpoint_1700000"
+#PjitPartitioner.num_partitions = 2
+utils.SaveCheckpointConfig.period = 5000
+utils.SaveCheckpointConfig.keep = 3
+
+
+

balanced_bokmaal_nynorsk_lm_xxl.gin

@@ -0,0 +1,21 @@
+include 't5x/examples/t5/mt5/xxl.gin'
+include 'pretrain_cont.gin'
+#include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = "balanced_bokmaal_nynorsk_lm_stream"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_800_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+INITIAL_CHECKPOINT_PATH = "gs://north-t5x/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_t5x_xxl/checkpoint_1700000"
+#PjitPartitioner.num_partitions = 2
+utils.SaveCheckpointConfig.period = 5000
+utils.SaveCheckpointConfig.keep = 3
+
+
+

train_balanced_bokmaal_nynorsk_lm_base.sh

@@ -0,0 +1,9 @@
+PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://t5x-training/pretrained_models/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_pluss100k_lm_t5x_base/"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="balanced_bokmaal_nynorsk_lm_base.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \

train_balanced_bokmaal_nynorsk_lm_large.sh

@@ -0,0 +1,9 @@
+PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://t5x-training/pretrained_models/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_pluss100k_lm_t5x_large/"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="balanced_bokmaal_nynorsk_lm_large.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \

train_balanced_bokmaal_nynorsk_lm_small.sh

@@ -0,0 +1,9 @@
+PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://t5x-training/pretrained_models/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_pluss100k_lm_t5x_small/"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="balanced_bokmaal_nynorsk_lm_small.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \

train_balanced_bokmaal_nynorsk_lm_xl.sh

@@ -0,0 +1,9 @@
+PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://t5x-training/pretrained_models/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_pluss100k_lm_t5x_xl/"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="balanced_bokmaal_nynorsk_lm_xl.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \

train_balanced_bokmaal_nynorsk_lm_xxl.sh

@@ -0,0 +1,9 @@
+PROJECT_DIR=${HOME}"/models/pk-nb-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://t5x-training/pretrained_models/norwegian_NCC_plus_English_pluss200k_balanced_bokmaal_nynorsk_pluss100k_lm_t5x_xxl/"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="balanced_bokmaal_nynorsk_lm_xxl.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \