testing

README.md

@@ -0,0 +1,4 @@
+---
+license: apache-2.0
+---
+Evaluate nynorsk translateion

batch_nynorsk_NCC_base.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+PROJECT_DIR=${HOME}"/models/t5-nynorsk-evaluator"
+export PYTHONPATH=${PROJECT_DIR}
+#INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000\"
+#TRAIN_STEPS=1505000
+
+
+FINETUNE_STEPS=5000
+MODEL_BUCKET_DIR="gs://nb-t5x-us-central2/finetuned/"
+EVAL_PREFIX="ul2test/eval_nynorsk_"
+CHECKPOINT_BUCKET_DIR="gs://nb-t5x-us-central2/"
+CHECKPOINT_LIST=("exp1-t5-base-ul2-engvoc" "exp2-t5-base-ul2-scandvoc" "exp3-t5-base-span-engvoc" "exp4-t5-base-span-scandvoc" "exp5-t5-base-ul2-scandvoc-full" "exp6-t5-base-span-scandvoc-full" "exp7-t5-base-ul2-511-scandvoc" "exp8-t5-base-span-511-scandvoc" "exp9-t5-base-ul2-mt5voc" "exp10-t5-base-span-mt5voc" "exp11-t5-base-ul2-511-scandvoc-full" "exp12-t5-base-span-511-scandvoc-full" "exp13-t5-base-ul2-mt5voc-full" "exp14-t5-base-span-mt5voc-full" "exp15-t5-base-ul2-511-scandvoc-full-scratch" "exp16-t5-base-span-511-scandvoc-full-scratch" "exp17-t5-small-ul2-mt5voc-scratch" "exp18-t5-small-span-mt5voc-scratch" "exp19-t5-small-ul2-mt5voc" "exp20-t5-small-span-mt5voc" "exp21-t5-small-ul2-mt5voc-full" "exp22-t5-small-span-mt5voc-full") 
+
+
+PRETUNE_START_LIST=(100000 200000 300000 400000 500000 1000000 1100000 1184000 1200000 1204000 1284000 1300000 1384000 1400000 1484000 1500000)
+VERSION_LIST=("1" "2" "3" "4" "5") 
+
+index=$(($1 + 0))
+
+if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
+	echo "Error: You need to provide the number of the checkpoints below as a parameter."
+	for i in "${!CHECKPOINT_LIST[@]}"; do
+	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
+	done	
+	
+	  exit 1
+fi
+
+
+
+for v in "${VERSION_LIST[@]}"; do
+	for s in "${PRETUNE_START_LIST[@]}"; do
+	    INITIAL_CHECKPOINT_PATH="${CHECKPOINT_BUCKET_DIR}${CHECKPOINT_LIST[$((index-1))]}/checkpoint_${s}"
+	    TRAIN_STEPS=$((s+FINETUNE_STEPS))
+	    if [[ "$INITIAL_CHECKPOINT_PATH" == *"engvoc"* ]]; then
+		GIN_FILE="finetune_translate_base.gin"
+		MIXTURE_OR_TASK_NAME="translate_long"
+	    
+	    elif [[ "$INITIAL_CHECKPOINT_PATH" == *"scandvoc"* ]]; then
+		GIN_FILE="finetune_translate_base_scand.gin"
+		MIXTURE_OR_TASK_NAME="translate_long_scand"
+	    else
+	    	if [[ "$INITIAL_CHECKPOINT_PATH" == *"small"* ]]; then
+		   GIN_FILE="finetune_translate_small_mt5.gin"
+	   	else
+		   GIN_FILE="finetune_translate_base_mt5.gin"
+		fi
+		MIXTURE_OR_TASK_NAME="translate_long_mt5"
+	    fi
+
+	    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}v${v}_${CHECKPOINT_LIST[$((index-1))]}_${s}"
+	    command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
+	    echo "${command}"
+	    # Uncomment the next line to run the command:
+	    eval "${command}"
+    done
+done
+
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
+

batch_various.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+PROJECT_DIR=${HOME}"/models/t5-nynorsk-norbench"
+export PYTHONPATH=${PROJECT_DIR}
+
+FINETUNE_STEPS=5000
+EVAL_PREFIX="norbench/eval_translate_"
+MODEL_BUCKET_DIR="gs://pere-north-t5x/finetuned/"
+
+CHECKPOINT_LIST=(
+  "pere-north-t5x/pretrained_models/small/norwegian_NCC_plus_English_t5x_small/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/base/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+  "pere-north-t5x/pretrained_models/large/norwegian_NCC_plus_English_t5x_large/checkpoint_1500000"
+  "t5-data/pretrained_models/t5x/mt5_small/checkpoint_1000000"
+  "t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+  "t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000"
+)
+
+NAME_LIST=(
+  "north_t5_small_NCC"
+  "north_t5_base_NCC"
+  "north_t5_large_NCC"
+  "mt5_small"
+  "mt5_base"
+  "mt5_large"
+)
+
+TASK_LIST=("translate_long_mt5" "translate_long_mt5" "translate_long_mt5" "translate_long_mt5" "translate_long_mt5" "translate_long_mt5")
+
+GIN_LIST=(
+  "finetune_translate_small_mt5.gin"
+  "finetune_translate_base_mt5.gin"
+  "finetune_translate_large_mt5.gin"
+  "finetune_translate_small_mt5.gin"
+  "finetune_translate_base_mt5.gin"
+  "finetune_translate_large_mt5.gin"
+)
+
+START_LIST=(1000000 1000000 1000000 1500000 1500000 1500000)
+EXP_LIST=(115 116 117 118 119 120)
+
+
+VERSION_LIST=("1" "2" "3" "4" "5") 
+
+index=$(($1 + 0))
+
+if [ $index -lt 1 ] || [ $index -gt ${#CHECKPOINT_LIST[@]} ] || [ $# -ne 1 ]; then
+	echo "Error: You need to provide the number of the checkpoints below as a parameter."
+	for i in "${!CHECKPOINT_LIST[@]}"; do
+	        echo "$((i+1)). ${CHECKPOINT_LIST[i]}"
+	done	
+	
+	  exit 1
+fi
+
+for v in "${VERSION_LIST[@]}"; do
+	i=($index-1)
+    INITIAL_CHECKPOINT_PATH="gs://${CHECKPOINT_LIST[i]}"
+    
+    TRAIN_STEPS=$((START_LIST[i]+FINETUNE_STEPS))
+    GIN_FILE=${GIN_LIST[i]}
+    MIXTURE_OR_TASK_NAME=${TASK_LIST[i]}
+    MODEL_DIR="${MODEL_BUCKET_DIR}${EVAL_PREFIX}exp${EXP_LIST[i]}_${NAME_LIST[i]}_v${v}"
+
+    command="python3 ../../t5x/t5x/train.py --gin_search_paths=\"./\" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file=${GIN_FILE} --gin.INITIAL_CHECKPOINT_PATH=\\\"${INITIAL_CHECKPOINT_PATH}\\\" --gin.MIXTURE_OR_TASK_NAME=\\\"${MIXTURE_OR_TASK_NAME}\\\" --gin.MODEL_DIR=\\\"${MODEL_DIR}\\\""
+    echo "${command}"
+    # Uncomment the next line to run the command:
+    eval "${command}"
+done
+
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
+#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate_long\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
+

finetune_translate_base.gin

@@ -0,0 +1,33 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/t5_1_1/base.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

finetune_translate_base_mt5.gin

@@ -0,0 +1,34 @@
+from __gin__ import dynamic_registration
+import tasks
+import seqio
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include 't5x/examples/t5/mt5/base.gin'
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

finetune_translate_base_scand.gin

@@ -0,0 +1,37 @@
+from __gin__ import dynamic_registration
+import tasks
+import seqio
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/t5_1_1/base.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+VOCABULARY = @seqio.SentencePieceVocabulary()
+seqio.SentencePieceVocabulary.sentencepiece_model_file = "gs://nb-t5/t5/vocabs/wikipedia/no-da-en-sv-nn-is_32000_unigram.sp.model"
+seqio.SentencePieceVocabulary.extra_ids = 100
+

finetune_translate_large.gin

@@ -0,0 +1,33 @@
+from __gin__ import dynamic_registration
+import tasks
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include "t5x/examples/t5/t5_1_1/large.gin"
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

finetune_translate_large_mt5.gin

@@ -0,0 +1,34 @@
+from __gin__ import dynamic_registration
+import tasks
+import seqio
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include 't5x/examples/t5/mt5/large.gin'
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

finetune_translate_small_mt5.gin

@@ -0,0 +1,34 @@
+from __gin__ import dynamic_registration
+import tasks
+import seqio
+
+import __main__ as train_script
+from t5.data import mixtures
+from t5x import models
+from t5x import partitioning
+from t5x import utils
+
+include 't5x/examples/t5/mt5/small.gin'
+include "t5x/configs/runs/finetune.gin"
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+INITIAL_CHECKPOINT_PATH = %gin.REQUIRED
+TRAIN_STEPS = %gin.REQUIRED  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+USE_CACHED_TASKS = False
+DROPOUT_RATE = 0.1
+RANDOM_SEED = 0
+
+#Fixing a small error
+infer_eval/utils.DatasetConfig:
+  task_feature_lengths = %TASK_FEATURE_LENGTHS
+
+#Saving every 1000 steps
+utils.SaveCheckpointConfig:
+  period = 1000
+  keep = 1  # number of checkpoints to keep
+
+# Might have to ba changed based on architecture
+# partitioning.PjitPartitioner.num_partitions = 1
+
+

generate_stats.py

@@ -0,0 +1,112 @@
+from google.cloud import storage
+import pandas as pd
+import json
+import re
+import sys
+
+# Create a storage client
+client = storage.Client()
+
+# Get the bucket
+bucket_name = "nb-t5x-us-central2"
+bucket = client.bucket(bucket_name)
+
+
+#checkpoints=["exp1-t5-base-ul2-engvoc","exp2-t5-base-ul2-scandvoc","exp3-t5-base-span-engvoc","exp4-t5-base-span-scandvoc","exp5-t5-base-ul2-scandvoc-full","exp6-t5-base-span-scandvoc-full","exp7-t5-base-ul2-511-scandvoc","exp8-t5-base-span-511-scandvoc","exp9-t5-base-ul2-mt5voc","exp10-t5-base-span-mt5voc","exp11-t5-base-ul2-511-scandvoc-full","exp12-t5-base-span-511-scandvoc-full","exp13-t5-base-ul2-mt5voc-full","exp14-t5-base-span-mt5voc-full","exp14-t5-base-span-mt5voc-full","exp15-t5-base-ul2-511-scandvoc-full-scratch","exp16-t5-base-span-511-scandvoc-full-scratch","exp17-t5-small-ul2-mt5voc-scratch","exp18-t5-small-span-mt5voc-scratch","exp19-t5-small-ul2-mt5voc","exp20-t5-small-span-mt5voc","exp21-t5-small-ul2-mt5voc-full","exp22-t5-small-span-mt5voc-full"] 
+
+checkpoints=["exp1-t5-base-ul2-engvoc","exp2-t5-base-ul2-scandvoc","exp3-t5-base-span-engvoc","exp4-t5-base-span-scandvoc","exp5-t5-base-ul2-scandvoc-full","exp6-t5-base-span-scandvoc-full","exp7-t5-base-ul2-511-scandvoc","exp8-t5-base-span-511-scandvoc","exp9-t5-base-ul2-mt5voc","exp10-t5-base-span-mt5voc","exp11-t5-base-ul2-511-scandvoc-full","exp12-t5-base-span-511-scandvoc-full","exp13-t5-base-ul2-mt5voc-full","exp14-t5-base-span-mt5voc-full","exp15-t5-base-ul2-511-scandvoc-full-scratch","exp16-t5-base-span-511-scandvoc-full-scratch","exp17-t5-small-ul2-mt5voc-scratch","exp18-t5-small-span-mt5voc-scratch","exp19-t5-small-ul2-mt5voc","exp20-t5-small-span-mt5voc","exp21-t5-small-ul2-mt5voc-full","exp22-t5-small-span-mt5voc-full"] 
+
+start=["100000","200000","300000","400000","500000","1000000","1100000","1184000","1200000","1204000","1284000","1300000","1384000","1400000","1484000","1500000"]
+
+iterations=["1","2","3","4","5"]
+file_names=[]
+
+for i in iterations:
+    for c in checkpoints:
+        for s in start:
+            if "scand" in c:
+                name = f'finetuned/ul2test/eval_nynorsk_v{i}_{c}_{s}/inference_eval/translate_long_scand-metrics.jsonl'
+            elif "mt5" in c:
+                name = f'finetuned/ul2test/eval_nynorsk_v{i}_{c}_{s}/inference_eval/translate_long_mt5-metrics.jsonl'
+            else:
+                name = f'finetuned/ul2test/eval_nynorsk_v{i}_{c}_{s}/inference_eval/translate_long-metrics.jsonl'
+            file_names.append(name)
+
+
+#list to store json files data
+file_contents = []
+
+downloaded = 0
+not_downloaded = 0
+
+#print(file_names)
+#print(bucket)
+#sys.exit(-1)
+
+#iterate over the files
+for file_name in file_names:
+    # Get the file
+    blob = bucket.get_blob(file_name)
+    print(f'gs://{bucket_name}/{file_name}')
+    
+    if not blob:
+        #print(f"Unable to download {file_name}")
+        not_downloaded+=1
+        continue
+    else:
+        downloaded+=1
+
+    content = blob.download_as_string().decode("utf-8")
+    # Split the content by newline
+    lines = content.split("\n")
+    
+    #iterate over the lines
+    for n,line in enumerate(lines):
+        if not line:
+            continue
+        #print(line)
+        #print(file_name)
+        data = json.loads(line)
+        data['base_file_name'] = file_name
+        pretraining_steps = re.search(r"(voc_|voc-full_|voc-full-scratch_|voc-scratch_)(.*?)(?=/)", file_name).group(2)
+        data['pretraining_steps'] = int(pretraining_steps)
+        data['finetuning_steps'] = data['step'] - int(pretraining_steps)
+        data['vocab'] = re.search(r"-(\w+?)voc", file_name).group(1)
+        data['experiment'] = re.search(r"_exp(\w+?)-", file_name).group(1) 
+        data['version'] = re.search(r"_v(\w+?)_exp", file_name).group(1) 
+        data['experiment_name'] = re.search(r"exp\d+-(.*?)_", file_name).group(1) 
+        file_contents.append(data)
+
+print(f"\nTotally {downloaded} files downloaded, {not_downloaded} files not downloaded")
+
+df = pd.json_normalize(file_contents)
+only_5000 = df[df["finetuning_steps"] == 5000]
+grouped = only_5000[["experiment_name","experiment","pretraining_steps", "accuracy", "f1_macro", "bleu"]].groupby(["experiment","experiment_name","pretraining_steps"])
+average_at_5000 = grouped.mean().reset_index()
+average_at_5000 = average_at_5000.assign(num_experiments=grouped.size().values)
+
+only_3000 = df[df["finetuning_steps"] == 3000]
+grouped = only_3000[["experiment_name","experiment","pretraining_steps", "accuracy", "f1_macro", "bleu"]].groupby(["experiment","experiment_name","pretraining_steps"])
+average_at_3000 = grouped.mean().reset_index()
+average_at_3000 = average_at_3000.assign(rows_count=grouped.size().values)
+
+#print(average_at_3000.to_string(index=False))
+print(average_at_5000.to_string(index=False))
+
+print("\nNot complete:")
+uncomplete = average_at_5000[average_at_5000['num_experiments'] != 5]
+print(uncomplete)
+
+
+df.to_json("stats/all.jsonl", orient="records", lines=True)
+df.to_csv("stats/all.csv", index=False)
+
+only_5000.to_json("stats/only_5000.jsonl", orient="records", lines=True)
+only_5000.to_csv("stats/only_5000.csv", index=False)
+
+average_at_5000.to_json("stats/average_at_5000.jsonl", orient="records", lines=True)
+average_at_5000.to_csv("stats/average_at_5000.csv", index=False)
+
+
+print(f"Files exported to stats")
+

generate_stats_various.py

@@ -0,0 +1,116 @@
+from google.cloud import storage
+import pandas as pd
+import json
+import re
+import sys
+
+# Create a storage client
+client = storage.Client()
+
+# Get the bucket
+bucket_name = "nb-t5x-us-central2"
+bucket = client.bucket(bucket_name)
+
+
+checkpoints=["exp115_mt5_small","exp116_north_t5_base_NCC","exp117_north_t5_base_NCC_lm","exp118_north_t5_base_scand3M","exp119_mt5_base","exp120_sab_base_2","exp121_sab_base_3","exp122_sab_base_4"]
+
+
+start=["1500000","1600000","2000000","3000000","4000000"]
+
+iterations=["1","2","3","4","5"]
+file_names=[]
+
+for i in iterations:
+    for c in checkpoints:
+        for s in start:
+            if "scand" in c:
+                name = f'finetuned/ul2test/eval_nynorsk_{c}_v{i}/inference_eval/translate_full_scand-metrics.jsonl'
+            elif ("mt5" in c) or ("north" in c):
+                name = f'finetuned/ul2test/eval_nynorsk_{c}_v{i}/inference_eval/translate_full_mt5-metrics.jsonl'
+            else:
+                name = f'finetuned/ul2test/eval_nynorsk_{c}_v{i}/inference_eval/translate_full-metrics.jsonl'
+            file_names.append(name)
+
+
+#list to store json files data
+file_contents = []
+
+downloaded = 0
+not_downloaded = 0
+
+#print(file_names)
+#print(bucket)
+#sys.exit(-1)
+
+#iterate over the files
+for file_name in file_names:
+    # Get the file
+    blob = bucket.get_blob(file_name)
+    print(f'gs://{bucket_name}/{file_name}')
+
+    if not blob:
+        #print(f"Unable to download {file_name}")
+        not_downloaded+=1
+        continue
+    else:
+        downloaded+=1
+
+    content = blob.download_as_string().decode("utf-8")
+    
+    #print(file_name)
+    #print(content)
+
+    # Split the content by newline
+    lines = content.split("\n")
+    
+    #iterate over the lines
+    for n,line in enumerate(lines):
+        if not line:
+            continue
+        #print(line)
+        #print(file_name)
+        data = json.loads(line)
+        data['base_file_name'] = file_name
+        pretraining_steps = 0 #re.search(r"(voc_|voc-full_|voc-full-scratch_|voc-scratch_)(.*?)(?=/)", file_name).group(2)
+        data['pretraining_steps'] = int(pretraining_steps)
+        data['finetuning_steps'] = int(str(data['step'])[-4:])
+        data['vocab'] = re.search(r"_(\w+?)-metric", file_name).group(1)
+        data['experiment'] = re.search(r"_exp(\w+?)_", file_name).group(1) 
+        data['version'] = re.search(r"_v(\w+?)/", file_name).group(1) 
+        data['experiment_name'] = re.search(r"exp\d+_(.*?)_v", file_name).group(1) 
+        file_contents.append(data)
+
+print(f"\nTotally {downloaded} files downloaded, {not_downloaded} files not downloaded")
+
+df = pd.json_normalize(file_contents)
+df = df.drop_duplicates(subset=['step','experiment','version']).reset_index()
+only_5000 = df[df["finetuning_steps"] == 5000]
+grouped_at_5000 = only_5000[["experiment_name","experiment","pretraining_steps", "accuracy", "f1_macro"]].groupby(["experiment","experiment_name","pretraining_steps"])
+average_at_5000 = grouped_at_5000.mean().reset_index()
+average_at_5000 = average_at_5000.assign(num_experiments=grouped_at_5000.size().values)
+only_3000 = df[df["finetuning_steps"] == 3000]
+grouped = only_3000[["experiment_name","experiment","pretraining_steps", "accuracy", "f1_macro"]].groupby(["experiment","experiment_name","pretraining_steps"])
+average_at_3000 = grouped.mean().reset_index()
+average_at_3000 = average_at_3000.assign(rows_count=grouped.size().values)
+
+#print(average_at_3000.to_string(index=False))
+print(average_at_5000.to_string(index=False))
+
+print("\nNot complete:")
+uncomplete = average_at_5000[average_at_5000['num_experiments'] != 5]
+print(uncomplete)
+
+df.to_json("stats_various/all.jsonl", orient="records", lines=True)
+df.to_csv("stats_various/all.csv", index=False)
+
+only_5000.to_json("stats_various/only_5000.jsonl", orient="records", lines=True)
+only_5000.to_csv("stats_various/only_5000.csv", index=False)
+
+average_at_5000.to_json("stats_various/average_at_5000.jsonl", orient="records", lines=True)
+average_at_5000.to_csv("stats_various/average_at_5000.csv", index=False)
+
+
+print(f"Files exported to stats")
+
+
+

my_metrics.py

@@ -0,0 +1,7 @@
+import sklearn.metrics
+import numpy as np
+
+def f1_macro(targets, predictions):
+      targets, predictions = np.asarray(targets).astype(str), np.asarray(predictions).astype(str)
+      return {"f1_macro": 100*sklearn.metrics.f1_score(targets, predictions, average='macro')}
+

nb_nn_10000.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13a5bc673760b418473cc6c7636746d531e6ae261879720ae2ab081e4c08c404
+size 2063855

tasks.py

@@ -0,0 +1,229 @@
+# /home/perk/mymodel/categorisation-mt5x/tasks.py
+
+
+import functools
+import seqio
+import my_metrics
+import tensorflow_datasets as tfds
+from t5.evaluation import metrics
+from t5.data import preprocessors
+#import my_preprocessors
+import t5
+import tensorflow.compat.v1 as tf
+
+
+
+tsv_parliament_path = {
+        "train": "gs://notram-public/finetune_datasets/parliament_speeches_1998_2016_frp_or_sv/train.tsv",
+        "validation": "gs://notram-public/finetune_datasets/parliament_speeches_1998_2016_frp_or_sv/dev.tsv",
+        "test": "gs://notram-public/finetune_datasets/parliament_speeches_1998_2016_frp_or_sv/test.tsv"
+}
+
+tsv_translate_path = {
+        "train": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/train.tsv",
+        "validation": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/dev.tsv",
+        "test": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/test.tsv"
+}
+
+tsv_translate_long_path = {
+        "train": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/train_long.tsv",
+        "validation": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/dev.tsv",
+        "test": "gs://nb-t5x-us-central2/corpus_bokmal_nynorsk/test.tsv"
+}
+
+tsv_sentiment_path = {
+        "train": "gs://notram-public/finetune_datasets/norec_sentiment/train.tsv",
+        "validation": "gs://notram-public/finetune_datasets/norec_sentiment/dev.tsv",
+        "test": "gs://notram-public/finetune_datasets/norec_sentiment/test.tsv"
+}
+
+json_angry_tweets_path = {
+        "train": "gs://notram-public/finetune_datasets/angry_tweets/train.jsonl",
+        "validation": "gs://notram-public/finetune_datasets/angry_tweets/test.jsonl",
+        "test": "gs://notram-public/finetune_datasets/angry_tweets/test.jsonl"
+}
+
+tsv_angry_tweets_path = {
+        "train": "gs://notram-public/finetune_datasets/angry_tweets/train.tsv",
+        "validation": "gs://notram-public/finetune_datasets/angry_tweets/test.tsv",
+        "test": "gs://notram-public/finetune_datasets/angry_tweets/test.tsv"
+}
+
+
+tsv_dane_path = {
+        "train": "gs://notram-public/finetune_datasets/dane/train.tsv",
+        "validation": "gs://notram-public/finetune_datasets/dane/test.tsv",
+        "test": "gs://notram-public/finetune_datasets/dane/test.tsv"
+}
+
+tsv_dane_tokens_path = {
+        "train": "gs://notram-public/finetune_datasets/dane/train_tokens.tsv",
+        "validation": "gs://notram-public/finetune_datasets/dane/test_tokens.tsv",
+        "test": "gs://notram-public/finetune_datasets/dane/test_tokens.tsv"
+}
+
+
+tsv_dane_long_tokens_path = {
+        "train": "gs://notram-public/finetune_datasets/dane/train_long_tokens.tsv",
+        "validation": "gs://notram-public/finetune_datasets/dane/test_long_tokens.tsv",
+        "test": "gs://notram-public/finetune_datasets/dane/test_long_tokens.tsv"
+}
+
+
+#vocabulary = seqio.SentencePieceVocabulary(
+#                'gs://t5-data/vocabs/mc4.250000.100extra/sentencepiece.model', extra_ids=0)
+scand_vocabulary=seqio.SentencePieceVocabulary('gs://nb-t5/t5/vocabs/wikipedia/no-da-en-sv-nn-is_32000_unigram.sp.model', extra_ids=100)
+eng_vocabulary=seqio.SentencePieceVocabulary('gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model', extra_ids=0)
+mt5_vocabulary=seqio.SentencePieceVocabulary('gs://t5-data/vocabs/mc4.250000.100extra/sentencepiece.model', extra_ids=0)
+
+DEFAULT_OUTPUT_FEATURES = {
+            "inputs": seqio.Feature(
+                        vocabulary=eng_vocabulary, add_eos=True,
+                                required=False),
+                "targets": seqio.Feature(
+                            vocabulary=eng_vocabulary, add_eos=True)
+                }
+
+
+SCAND_OUTPUT_FEATURES = {
+            "inputs": seqio.Feature(
+                        vocabulary=scand_vocabulary, add_eos=True,
+                                required=False),
+                "targets": seqio.Feature(
+                            vocabulary=scand_vocabulary, add_eos=True)
+                }
+
+MT5_OUTPUT_FEATURES = {
+    "inputs": seqio.Feature(
+        vocabulary=mt5_vocabulary, add_eos=True,
+        required=False),
+    "targets": seqio.Feature(
+        vocabulary=mt5_vocabulary, add_eos=True)
+}
+
+
+
+def categorise_preprocessor(ds):
+  def normalize_text(text):
+    """Lowercase and remove quotes from a TensorFlow string."""
+    #text = tf.strings.regex_replace(text,"'(.*)'", r"\1")
+    ...
+    return text
+
+  def to_inputs_and_targets(ex):
+    """Map {"source": ..., "source": ...}->{"target": ..., "target": ...}."""
+    return {
+        "inputs":
+             tf.strings.join(
+                 [normalize_text(ex["source"])]),
+        "targets": 
+	    tf.strings.join(
+                 [normalize_text(ex["target"])]),
+    }
+  return ds.map(to_inputs_and_targets, 
+                num_parallel_calls=tf.data.experimental.AUTOTUNE)
+
+
+seqio.TaskRegistry.add(
+    "parliament",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_parliament_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["target","source"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro],
+    output_features=DEFAULT_OUTPUT_FEATURES,
+)   
+
+seqio.TaskRegistry.add(
+    "sentiment",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_sentiment_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["target","source"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro],
+    output_features=DEFAULT_OUTPUT_FEATURES,
+)   
+
+
+seqio.TaskRegistry.add(
+    "translate",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_translate_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["source","target"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro,metrics.bleu],
+    output_features=DEFAULT_OUTPUT_FEATURES,
+)   
+
+seqio.TaskRegistry.add(
+    "translate_long_mt5",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_translate_long_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["source","target"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro,metrics.bleu],
+    output_features=MT5_OUTPUT_FEATURES,
+)   
+
+seqio.TaskRegistry.add(
+    "translate_long_scand",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_translate_long_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["source","target"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro,metrics.bleu],
+    output_features=SCAND_OUTPUT_FEATURES,
+)   
+
+seqio.TaskRegistry.add(
+    "translate_long",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=tsv_translate_long_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          t5.data.preprocessors.parse_tsv,
+          field_names=["source","target"]),
+      categorise_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro,metrics.bleu],
+    output_features=DEFAULT_OUTPUT_FEATURES,
+)   
+