a lot of small changes so theat eval seems to run fine

eval_base.sh

@@ -1,11 +1,13 @@
-PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
-#EVAL_OUTPUT_DIR="gs://nb-t5x/eval/"
-T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+#PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+#T5X_DIR="../../t5x"  # directory where the t5x is cloned.
 CHECKPOINT_PATH="gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000"
-export PYTHONPATH=${PROJECT_DIR}
+#export PYTHONPATH=${PROJECT_DIR}
 
 python3 eval.py \
-  --gin_search_paths=${PROJECT_DIR} \
+  --gin_search_paths="./" \
   --gin_file="eval_categorisation_base.gin" \
-  --gin.CHECKPOINT_PATH=\"${CHECKPOINT_PATH}\" \
-#  --gin.EVAL_OUTPUT_DIR=\"${EVAL_OUTPUT_DIR}\" \
+  --gin.SPLIT=\"validation\" \
+  --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1505000\" \
+
+#"gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000" \
+#--gin.SPLIT="validation" \  

eval_categorisation_base.gin

@@ -9,8 +9,8 @@ from t5x import utils
 include "t5x/examples/t5/mt5/base.gin"
 
 CHECKPOINT_PATH = %gin.REQUIRED  # passed via commandline
+SPLIT = %gin.REQUIRED  # passed via commandline
 EVAL_OUTPUT_DIR = "./log/"
-
 DROPOUT_RATE = 0.0  # unused boilerplate
 MIXTURE_OR_TASK_NAME = "categorise"
 
@@ -24,8 +24,8 @@ eval_script.evaluate:
 utils.DatasetConfig:
   mixture_or_task_name = %MIXTURE_OR_TASK_NAME
   task_feature_lengths = {"inputs": 512, "targets": 2}
-  split = 'validation'
-  batch_size = 32
+  split = %SPLIT
+  batch_size = 16
   shuffle = False
   seed = 42
 

finetune_categorisation_base.gin

@@ -21,31 +21,19 @@ RANDOM_SEED = 0
 infer_eval/utils.DatasetConfig:
   task_feature_lengths = %TASK_FEATURE_LENGTHS
 
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+
+INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+
 # Pere: Only necessary if we load a t5 model. We can start with an t5x model here
 # `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
 # using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
 # set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
 # `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
-#LOSS_NORMALIZING_FACTOR = 234496
-
-#INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1/checkpoint_1100000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1360000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1_lr_1/checkpoint_1100000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_scandinavian/checkpoint_1100000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_2000000"
-INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+# LOSS_NORMALIZING_FACTOR = 234496
 
-
-
-#train_script.train:
-#  eval_period = 500
-#  partitioner = @partitioning.ModelBasedPjitPartitioner()
+# Might have to ba chaned based on architecture
 # partitioning.PjitPartitioner.num_partitions = 1
 
-# `num_decodes` is equivalent to a beam size in a beam search decoding.
-# models.EncoderDecoderModel.predict_batch_with_aux.num_decodes = 1
-
-#mesh_transformer.learning_rate_schedules.constant_learning_rate.learning_rate = 0.0005
-#run.learning_rate_schedule = @learning_rate_schedules.constant_learning_rate
-

log/config.gin

@@ -11,7 +11,8 @@ import tasks
 
 # Macros:
 # ==============================================================================
-CHECKPOINT_PATH = 'gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000'
+CHECKPOINT_PATH = \
+    'gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1505000'
 DROPOUT_RATE = 0.0
 EVAL_OUTPUT_DIR = './log/'
 LABEL_SMOOTHING = 0.0
@@ -19,6 +20,7 @@ LOSS_NORMALIZING_FACTOR = None
 MIXTURE_OR_TASK_NAME = 'categorise'
 MODEL = @models.EncoderDecoderModel()
 OPTIMIZER = @adafactor.Adafactor()
+SPLIT = 'validation'
 VOCABULARY = @seqio.SentencePieceVocabulary()
 Z_LOSS = 0.0001
 
@@ -31,11 +33,11 @@ adafactor.Adafactor.step_offset = 0
 
 # Parameters for utils.DatasetConfig:
 # ==============================================================================
-utils.DatasetConfig.batch_size = 32
+utils.DatasetConfig.batch_size = 16
 utils.DatasetConfig.mixture_or_task_name = %MIXTURE_OR_TASK_NAME
 utils.DatasetConfig.seed = 42
 utils.DatasetConfig.shuffle = False
-utils.DatasetConfig.split = 'validation'
+utils.DatasetConfig.split = %SPLIT
 utils.DatasetConfig.task_feature_lengths = {'inputs': 512, 'targets': 2}
 
 # Parameters for models.EncoderDecoderModel:

log/eval_results_t1v-n-7b23714e-w-0.jsonl

@@ -1,7 +1,5 @@
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "08-04-2022 12:06:19", "split": "validation", "result": {"accuracy": 86.33333333333333}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "08-04-2022 15:52:57", "task": "categorise", "feature_length": null, "split": "validation", "result": {"accuracy": 86.33333333333333, "f1_macro": 86.33090327169275}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "08-04-2022 17:40:27", "task": "categorise", "feature_length": null, "split": "validation", "result": {"accuracy": 86.33333333333333, "f1_macro": 86.33090327169275}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "08-04-2022 18:07:14", "task": "categorise", "feature_length": null, "split": "validation", "result": {"accuracy": 86.33333333333333, "f1_macro": 86.33090327169275}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "08-04-2022 18:31:25", "task": "categorise", "feature_length": {"inputs": 512, "targets": 2}, "split": "validation", "result": {"accuracy": 84.83333333333334, "f1_macro": 84.82911919977771}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "11-04-2022 06:48:47", "split": "validation", "result": {"accuracy": 84.83333333333334}}
-{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "eval_date": "11-04-2022 07:01:50", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 32, "result": {"accuracy": 84.83333333333334, "f1_macro": 84.82911919977771}}
+{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "task": "categorise", "eval_date": "11-04-2022 07:28:10", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.08333333333333, "f1_macro": 85.07959287015703}}
+{"model": "gs://nb-t5x/eval_norwegian_NCC_2_000_000/checkpoint_2005000", "task": "categorise", "eval_date": "11-04-2022 07:44:00", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 84.41666666666666, "f1_macro": 84.40877360830586}}
+{"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1510000", "task": "categorise", "eval_date": "11-04-2022 07:49:05", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 86.5, "f1_macro": 86.49816224986179}}
+{"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1510000", "task": "categorise", "eval_date": "11-04-2022 07:51:08", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.25, "f1_macro": 85.2425290303216}}
+{"model": "gs://nb-t5x-us-central2/finetuned/norwegian_NCC_pluss_english_1_500_000/checkpoint_1505000", "task": "categorise", "eval_date": "11-04-2022 08:13:36", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.25, "f1_macro": 85.24014985000407}}