all batches

batch_parliament_large.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+export PYTHONPATH=${PROJECT_DIR}
+INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_large/checkpoint_1500000\"
+TRAIN_STEPS=1505000
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_large.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_large_v1\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_large.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_large_v2\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_large.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_large_v3\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_large.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_large_v4\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_large.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_large_v5\"
+
+

batch_parliament_small.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+export PYTHONPATH=${PROJECT_DIR}
+INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_small/checkpoint_1500000\"
+TRAIN_STEPS=1505000
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_small.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_small_v1\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_small.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_small_v2\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_small.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_small_v3\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_small.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_small_v4\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_small.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_small_v5\"
+
+

batch_parliament_xl.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+export PYTHONPATH=${PROJECT_DIR}
+INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_xl/checkpoint_1500000\"
+TRAIN_STEPS=1505000
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xl_v1\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xl_v2\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xl_v3\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xl_v4\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xl_v5\"
+
+

batch_parliament_xxl.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+export PYTHONPATH=${PROJECT_DIR}
+INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_xxl/checkpoint_1500000\"
+TRAIN_STEPS=1505000
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xxl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xxl_v1\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xxl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xxl_v2\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xxl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xxl_v3\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xxl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xxl_v4\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_categorisation_xxl.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/parliament_NCC_xxl_v5\"
+
+

finetune_categorisation_large.gin

@@ -10,42 +10,30 @@ from t5x import utils
 include "t5x/examples/t5/mt5/large.gin"
 include "t5x/configs/runs/finetune.gin"
 
-MIXTURE_OR_TASK_NAME = "dane"
-TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 512}
-TRAIN_STEPS = 1_720_000  # 1700000 pre-trained steps + 20000 fine-tuning steps.
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED #"gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+TRAIN_STEPS = %gin.REQUIRED #1_010_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
 USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
 
-# Pere: Only necessary if we load a t5 model. We can start with an t5x model here
-# `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
-# using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
-# set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
-# `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
-#LOSS_NORMALIZING_FACTOR = 234496
-
-INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_pluss200k_scandinavian_t5x_large/checkpoint_1700000"
-
-
 #Fixing a small error
 infer_eval/utils.DatasetConfig:
   task_feature_lengths = %TASK_FEATURE_LENGTHS
 
-#Saving every 2000 steps
+#Saving every 1000 steps
 utils.SaveCheckpointConfig:
-  period = 2000
+  period = 1000
 
 
+# Pere: Only necessary if we load a t5 model. We can start with an t5x model here
+# `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
+# using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
+# set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
+# `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
+# LOSS_NORMALIZING_FACTOR = 234496
 
-
-#train_script.train:
-#  eval_period = 500
-#  partitioner = @partitioning.ModelBasedPjitPartitioner()
-partitioning.PjitPartitioner.num_partitions = 1
-
-# `num_decodes` is equivalent to a beam size in a beam search decoding.
-# models.EncoderDecoderModel.predict_batch_with_aux.num_decodes = 1
-
-#mesh_transformer.learning_rate_schedules.constant_learning_rate.learning_rate = 0.0005
-#run.learning_rate_schedule = @learning_rate_schedules.constant_learning_rate
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
 

finetune_categorisation_small.gin

@@ -0,0 +1,39 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/mt5/small.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED #"gs://t5-data/pretrained_models/t5x/mt5_small/checkpoint_1000000"
+TRAIN_STEPS = %gin.REQUIRED #1_010_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+
+
+# Pere: Only necessary if we load a t5 model. We can start with an t5x model here
+# `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
+# using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
+# set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
+# `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
+# LOSS_NORMALIZING_FACTOR = 234496
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+

finetune_categorisation_xl.gin

@@ -0,0 +1,39 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/mt5/xl.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED #"gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+TRAIN_STEPS = %gin.REQUIRED #1_010_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+
+
+# Pere: Only necessary if we load a t5 model. We can start with an t5x model here
+# `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
+# using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
+# set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
+# `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
+# LOSS_NORMALIZING_FACTOR = 234496
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+

finetune_categorisation_xxl.gin

@@ -0,0 +1,39 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/mt5/xxl.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED #"gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+TRAIN_STEPS = %gin.REQUIRED #1_010_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+
+
+# Pere: Only necessary if we load a t5 model. We can start with an t5x model here
+# `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
+# using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
+# set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
+# `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
+# LOSS_NORMALIZING_FACTOR = 234496
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+