batch scripts

batch_sentiment_base.sh

@@ -1,21 +1,22 @@
 PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
 export PYTHONPATH=${PROJECT_DIR}
 
-python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\" &&
-python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\" &&
-python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\" &&
-python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\" &&
-python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v1_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v2_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v3_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v4_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v5_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v1_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v2_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v3_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v4_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/v5_finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+
 
 
 

eval_categorisation_base.gin

@@ -12,7 +12,7 @@ CHECKPOINT_PATH = %gin.REQUIRED  # passed via commandline
 SPLIT = %gin.REQUIRED  # passed via commandline
 EVAL_OUTPUT_DIR = "./log/"
 DROPOUT_RATE = 0.0  # unused boilerplate
-MIXTURE_OR_TASK_NAME = "parliament"
+MIXTURE_OR_TASK_NAME = "sentiment"
 
 eval_script.evaluate:
   model = %MODEL  # imported from separate gin file

log/config.gin

@@ -12,15 +12,15 @@ import tasks
 # Macros:
 # ==============================================================================
 CHECKPOINT_PATH = \
-    'gs://nb-t5x-us-central2/finetuned/norwegian_NCC_plus_English_t5x_base_1_500_000_parliament/checkpoint_1510000'
+    'gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000'
 DROPOUT_RATE = 0.0
 EVAL_OUTPUT_DIR = './log/'
 LABEL_SMOOTHING = 0.0
 LOSS_NORMALIZING_FACTOR = None
-MIXTURE_OR_TASK_NAME = 'parliament'
+MIXTURE_OR_TASK_NAME = 'sentiment'
 MODEL = @models.EncoderDecoderModel()
 OPTIMIZER = @adafactor.Adafactor()
-SPLIT = 'validation'
+SPLIT = 'test'
 VOCABULARY = @seqio.SentencePieceVocabulary()
 Z_LOSS = 0.0001
 

log/eval_results_t1v-n-7b23714e-w-0.jsonl

@@ -3,3 +3,17 @@
 {"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1510000", "task": "categorise", "eval_date": "11-04-2022 07:49:05", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 86.5, "f1_macro": 86.49816224986179}}
 {"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1510000", "task": "categorise", "eval_date": "11-04-2022 07:51:08", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.25, "f1_macro": 85.2425290303216}}
 {"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1505000", "task": "categorise", "eval_date": "11-04-2022 08:13:36", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.25, "f1_macro": 85.24014985000407}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:13:47", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 51.74999999999999, "f1_macro": 51.71777832289466}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:15:07", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 50.83333333333333, "f1_macro": 50.79383171490447}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:16:13", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 51.583333333333336, "f1_macro": 51.03489009889238}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:17:21", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 52.25, "f1_macro": 52.24253789654634}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:19:04", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 52.0, "f1_macro": 51.92307692307692}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:20:24", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 51.583333333333336, "f1_macro": 51.555040131076545}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "parliament", "eval_date": "13-04-2022 13:21:38", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 50.74999999999999, "f1_macro": 50.43938573350337}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 13:51:56", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 84.88372093023256, "f1_macro": 82.33497191011236}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 13:53:25", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.07751937984496, "f1_macro": 82.7653067863882}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 13:54:36", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.85271317829456, "f1_macro": 83.60657164617561}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:01:28", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 87.29016786570743, "f1_macro": 84.21729163839953}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:03:06", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 84.89208633093526, "f1_macro": 81.23942213621073}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:04:20", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.61151079136691, "f1_macro": 82.17948717948718}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:05:39", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 80.0959232613909, "f1_macro": 74.5979905029614}}

log/sentiment-metrics.jsonl

@@ -0,0 +1,10 @@
+{"step": 1501000, "accuracy": 71.12403100775194, "f1_macro": 51.34385976014934}
+{"step": 1502000, "accuracy": 85.65891472868216, "f1_macro": 83.4095135473331}
+{"step": 1503000, "accuracy": 85.65891472868216, "f1_macro": 83.24087078651685}
+{"step": 1504000, "accuracy": 85.65891472868216, "f1_macro": 83.4095135473331}
+{"step": 1505000, "accuracy": 86.04651162790698, "f1_macro": 83.57966197581501}
+{"step": 1506000, "accuracy": 85.85271317829456, "f1_macro": 83.60657164617561}
+{"step": 1507000, "accuracy": 85.27131782945736, "f1_macro": 82.90437329985352}
+{"step": 1508000, "accuracy": 85.46511627906976, "f1_macro": 83.10095324597064}
+{"step": 1509000, "accuracy": 84.88372093023256, "f1_macro": 82.5127304958376}
+{"step": 1510000, "accuracy": 85.27131782945736, "f1_macro": 82.72818561386819}