new norbench

my_metrics.py

@@ -0,0 +1,8 @@
+import sklearn.metrics
+import numpy as np
+
+def f1_macro(targets, predictions):
+    targets, predictions = np.asarray(targets), np.asarray(predictions)
+    unique_labels = np.unique(np.concatenate((targets, predictions)))
+    return {"f1_macro": 100 * sklearn.metrics.f1_score(targets, predictions, labels=unique_labels, average='macro')}
+

my_metrics_old.py

@@ -1,7 +0,0 @@
-import sklearn.metrics
-import numpy as np
-
-def f1_macro(targets, predictions):
-      targets, predictions = np.asarray(targets).astype(str), np.asarray(predictions).astype(str)
-      return {"f1_macro": 100*sklearn.metrics.f1_score(targets, predictions, average='macro')}
-

finetune_mt5_128.gin → norbench_sentence_sentiment_bs128.gin

@@ -21,7 +21,7 @@ USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
 BATCH_SIZE = 128
-EVAL_PERIOD = 316
+EVAL_PERIOD = 124
 
 #Fixing a small error
 infer_eval/utils.DatasetConfig:
@@ -29,7 +29,7 @@ infer_eval/utils.DatasetConfig:
 
 #Saving every 500 steps
 utils.SaveCheckpointConfig:
-  period = 316
+  period = 124
   keep = 1  # number of checkpoints to keep
 
 

batch_lrtest_sentence_sentiment_bs32.sh → norbench_sentence_sentiment_bs128.sh

@@ -5,7 +5,7 @@ echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
 
 
 FINETUNE_STEPS=5000
-EVAL_PREFIX="norbench/evalB_lr_semtiment_bs32"
+EVAL_PREFIX="norbench/norbenc_sentence_sentiment_bs128"
 MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
 
 CHECKPOINT_LIST=(
@@ -31,51 +31,30 @@ LR_LIST=(0.0001 0.0002 0.0004 0.0006 0.0008 0.001)
 
 
 GIN_LIST=(
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
-  "finetune_mt5_32.gin"
+  "norbench_sentence_sentiment_bs128.gin"
+  "norbench_sentence_sentiment_bs128.gin"
+  "norbench_sentence_sentiment_bs128.gin"
+  "norbench_sentence_sentiment_bs128.gin"
+  "norbench_sentence_sentiment_bs128.gin"
+  "norbench_sentence_sentiment_bs128.gin"
 )
 
 START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
-EXP_LIST=(901 902 903 904 905 906)
+EXP_LIST=(1 2 3 4 5 6)
 
 
-VERSION_LIST=("1") 
-
-index=$(($1 + 0))
-
-if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
-	echo "Error: You need to provide the number of the checkpoints below as a parameter."
-	for i in "${!CHECKPOINT_LIST[@]}"; do
-	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
-	done	
-	
-	  exit 1
-fi
-
-for v in "${VERSION_LIST[@]}"; do
-	i=($index-1)
+op through each experiment in EXP_LIST
+for i in "${!EXP_LIST[@]}"; do
     INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
-    
     TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
     GIN_FILE=${GIN_LIST[i]}
     MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
     LR=${LR_LIST[i]}
-    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
+    EXP=${EXP_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP}_${NAME_LIST[i]}"
 
     command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR}  --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
     echo "${command}"
     # Uncomment the next line to run the command:
-    eval "${command}"
+    # eval "${command}"
 done
-
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
-

norbench_sentence_sentiment_bs16.gin

@@ -0,0 +1,52 @@
+from __gin__ import dynamic_registration
+import tasks
+import seqio
+import optax
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include 't5x/examples/t5/mt5/base.gin'
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 256}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+LR = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+BATCH_SIZE = 16
+EVAL_PERIOD = 124
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 500 steps
+utils.SaveCheckpointConfig:
+  period = 124
+  keep = 1  # number of checkpoints to keep
+
+
+#optax.adamw.weight_decay = 0.1
+#OPTIMIZER = @optax.adamw
+#import t5x.optimizers
+#OPTIMIZER = @optax.adamw
+#optax.adamw.learning_rate = %LR
+#optax.adamw.weight_decay = 0.1
+
+
+utils.create_learning_rate_scheduler:
+  factors = 'constant'
+  base_learning_rate = %LR
+  warmup_steps = 1000
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

batch_lrtest_sentence_sentiment_bs64.sh → norbench_sentence_sentiment_bs16.sh

@@ -5,7 +5,7 @@ echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
 
 
 FINETUNE_STEPS=5000
-EVAL_PREFIX="norbench/evalB_lr_semtiment_bs64"
+EVAL_PREFIX="norbench/norbenc_sentence_sentiment_bs16"
 MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
 
 CHECKPOINT_LIST=(
@@ -31,51 +31,30 @@ LR_LIST=(0.0001 0.0002 0.0004 0.0006 0.0008 0.001)
 
 
 GIN_LIST=(
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
-  "finetune_mt5_64.gin"
+  "norbench_sentence_sentiment_bs16.gin"
+  "norbench_sentence_sentiment_bs16.gin"
+  "norbench_sentence_sentiment_bs16.gin"
+  "norbench_sentence_sentiment_bs16.gin"
+  "norbench_sentence_sentiment_bs16.gin"
+  "norbench_sentence_sentiment_bs16.gin"
 )
 
 START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
-EXP_LIST=(907 908 909 910 911 912)
+EXP_LIST=(1 2 3 4 5 6)
 
 
-VERSION_LIST=("1") 
-
-index=$(($1 + 0))
-
-if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
-	echo "Error: You need to provide the number of the checkpoints below as a parameter."
-	for i in "${!CHECKPOINT_LIST[@]}"; do
-	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
-	done	
-	
-	  exit 1
-fi
-
-for v in "${VERSION_LIST[@]}"; do
-	i=($index-1)
+op through each experiment in EXP_LIST
+for i in "${!EXP_LIST[@]}"; do
     INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
-    
     TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
     GIN_FILE=${GIN_LIST[i]}
     MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
     LR=${LR_LIST[i]}
-    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
+    EXP=${EXP_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP}_${NAME_LIST[i]}"
 
     command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR}  --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
     echo "${command}"
     # Uncomment the next line to run the command:
-    eval "${command}"
+    # eval "${command}"
 done
-
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
-

finetune_mt5_32.gin → norbench_sentence_sentiment_bs32.gin

@@ -21,7 +21,7 @@ USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
 BATCH_SIZE = 32
-EVAL_PERIOD = 316
+EVAL_PERIOD = 124
 
 #Fixing a small error
 infer_eval/utils.DatasetConfig:
@@ -29,7 +29,7 @@ infer_eval/utils.DatasetConfig:
 
 #Saving every 500 steps
 utils.SaveCheckpointConfig:
-  period = 316
+  period = 124
   keep = 1  # number of checkpoints to keep
 
 

batch_lrtest_sentence_sentiment_bs128.sh → norbench_sentence_sentiment_bs32.sh

@@ -4,8 +4,8 @@ export PYTHONPATH=${PROJECT_DIR}
 echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
 
 
-FINETUNE_STEPS=10000
-EVAL_PREFIX="norbench/evalB_lr_semtiment_bs128"
+FINETUNE_STEPS=5000
+EVAL_PREFIX="norbench/norbenc_sentence_sentiment_bs32"
 MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
 
 CHECKPOINT_LIST=(
@@ -31,51 +31,30 @@ LR_LIST=(0.0001 0.0002 0.0004 0.0006 0.0008 0.001)
 
 
 GIN_LIST=(
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
-  "finetune_mt5_128.gin"
+  "norbench_sentence_sentiment_bs32.gin"
+  "norbench_sentence_sentiment_bs32.gin"
+  "norbench_sentence_sentiment_bs32.gin"
+  "norbench_sentence_sentiment_bs32.gin"
+  "norbench_sentence_sentiment_bs32.gin"
+  "norbench_sentence_sentiment_bs32.gin"
 )
 
 START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
-EXP_LIST=(913 914 915 916 917 918)
+EXP_LIST=(1 2 3 4 5 6)
 
 
-VERSION_LIST=("1") 
-
-index=$(($1 + 0))
-
-if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
-	echo "Error: You need to provide the number of the checkpoints below as a parameter."
-	for i in "${!CHECKPOINT_LIST[@]}"; do
-	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
-	done	
-	
-	  exit 1
-fi
-
-for v in "${VERSION_LIST[@]}"; do
-	i=($index-1)
+op through each experiment in EXP_LIST
+for i in "${!EXP_LIST[@]}"; do
     INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
-    
     TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
     GIN_FILE=${GIN_LIST[i]}
     MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
     LR=${LR_LIST[i]}
-    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
+    EXP=${EXP_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP}_${NAME_LIST[i]}"
 
     command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR}  --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
     echo "${command}"
     # Uncomment the next line to run the command:
-    eval "${command}"
+    # eval "${command}"
 done
-
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
-

finetune_mt5_64.gin → norbench_sentence_sentiment_bs64.gin

@@ -21,7 +21,7 @@ USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0
 BATCH_SIZE = 64
-EVAL_PERIOD = 316
+EVAL_PERIOD = 124
 
 #Fixing a small error
 infer_eval/utils.DatasetConfig:
@@ -29,7 +29,7 @@ infer_eval/utils.DatasetConfig:
 
 #Saving every 500 steps
 utils.SaveCheckpointConfig:
-  period = 316
+  period = 124
   keep = 1  # number of checkpoints to keep
 
 

norbench_sentence_sentiment_bs64.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+PROJECT_DIR=${HOME}"/models/t5-nynorsk-norbench"
+export PYTHONPATH=${PROJECT_DIR}
+echo "PROJECT_DIR is set to: ${PROJECT_DIR}"
+
+
+FINETUNE_STEPS=5000
+EVAL_PREFIX="norbench/norbenc_sentence_sentiment_bs64"
+MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
+
+CHECKPOINT_LIST=(
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+)
+
+NAME_LIST=(
+  "north_t5_base_NCC_LR_0_0001"
+  "north_t5_base_NCC_LR_0_0002"
+  "north_t5_base_NCC_LR_0_0004"
+  "north_t5_base_NCC_LR_0_0006"
+  "north_t5_base_NCC_LR_0_0008"
+  "north_t5_base_NCC_LR_0_001"
+)
+
+TASK_LIST=("sentence_sentiment" "sentence_sentiment" "sentence_sentiment" "sentence_sentiment" "sentence_sentiment" "sentence_sentiment")
+LR_LIST=(0.0001 0.0002 0.0004 0.0006 0.0008 0.001)
+
+
+GIN_LIST=(
+  "norbench_sentence_sentiment_bs64.gin"
+  "norbench_sentence_sentiment_bs64.gin"
+  "norbench_sentence_sentiment_bs64.gin"
+  "norbench_sentence_sentiment_bs64.gin"
+  "norbench_sentence_sentiment_bs64.gin"
+  "norbench_sentence_sentiment_bs64.gin"
+)
+
+START_LIST=(1500000 1500000 1500000 1500000 1500000 1500000)
+EXP_LIST=(1 2 3 4 5 6)
+
+
+op through each experiment in EXP_LIST
+for i in "${!EXP_LIST[@]}"; do
+    INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
+    TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
+    GIN_FILE=${GIN_LIST[i]}
+    MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
+    LR=${LR_LIST[i]}
+    EXP=${EXP_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}_exp${EXP}_${NAME_LIST[i]}"
+
+    command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin.LR=${LR}  --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
+    echo "${command}"
+    # Uncomment the next line to run the command:
+    # eval "${command}"
+done

tasks.py

@@ -1,12 +1,12 @@
 # /home/perk/mymodel/categorisation-mt5x/tasks.py
 
-
 import functools
 import seqio
 import tensorflow_datasets as tfds
 from t5.evaluation import metrics
 from t5.data import preprocessors
 #import my_preprocessors
+import my_metrics
 import t5
 import tensorflow.compat.v1 as tf
 
@@ -193,7 +193,7 @@ seqio.TaskRegistry.add(
       categorise_preprocessor,
       seqio.preprocessors.tokenize_and_append_eos,
     ],
-    metric_fns=[metrics.accuracy,metrics.mean_multiclass_f1],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro],
     output_features=MT5_OUTPUT_FEATURES,
 )