update

finetune_categorisation_base.gin

@@ -12,7 +12,7 @@ include "t5x/configs/runs/finetune.gin"
 
 MIXTURE_OR_TASK_NAME = "categorise"
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
-TRAIN_STEPS = 1_299_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+TRAIN_STEPS = 1_635_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
 USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
@@ -26,9 +26,10 @@ RANDOM_SEED = 0
 
 #INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
 #INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1/checkpoint_1100000"
-INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1294000"
-
-
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1360000"
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1_lr_1/checkpoint_1100000"
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_scandinavian/checkpoint_1100000"
+INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1630000"
 
 #train_script.train:
 #  eval_period = 500

finetune_categorisation_large.gin

@@ -7,12 +7,12 @@ from t5x import models
 from t5x import partitioning
 from t5x import utils
 
-include "t5x/examples/t5/mt5/base.gin"
+include "t5x/examples/t5/mt5/large.gin"
 include "t5x/configs/runs/finetune.gin"
 
 MIXTURE_OR_TASK_NAME = "categorise"
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
-TRAIN_STEPS = 1_299_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+TRAIN_STEPS = 1_005_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
 USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
@@ -25,8 +25,9 @@ RANDOM_SEED = 0
 #LOSS_NORMALIZING_FACTOR = 234496
 
 #INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000"
 #INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1/checkpoint_1100000"
-INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1294000"
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1294000"
 
 
 

train_base.sh

@@ -1,7 +1,7 @@
 PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
 T5X_DIR="../../t5x"  # directory where the t5x is cloned.
 #Needs to be updated when moving to tpu-v4  it should then be in another zone
-MODEL_DIR="gs://nb-t5x/eval_v32_129k"
+MODEL_DIR="gs://nb-t5x/eval_norwegian_1_163_000"
 export PYTHONPATH=${PROJECT_DIR}
 
 python3 ${T5X_DIR}/t5x/train.py \

train_large.sh

@@ -0,0 +1,11 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+#Needs to be updated when moving to tpu-v4  it should then be in another zone
+MODEL_DIR="gs://nb-t5x/eval_large"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="finetune_categorisation_large.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" 
+