test

batch_parliament_base_ref.sh

@@ -0,0 +1,19 @@
+PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
+export PYTHONPATH=${PROJECT_DIR}
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v1_mt5x_base_1_000_000_parliament\" 
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v2_mt5x_base_1_000_000_parliament\" 
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v3_mt5x_base_1_000_000_parliament\" 
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v4_mt5x_base_1_000_000_parliament\" 
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"parliament\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v5_mt5x_base_1_000_000_parliament\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_t5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_mt5x_base_1_000_000_parliament/checkpoint_1010000\" 
+

batch_sentiment_base.sh

@@ -1,18 +1,18 @@
 PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
 export PYTHONPATH=${PROJECT_DIR}
 
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
 python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
-#python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin_file="finetune_categorisation_base.gin" --gin.MIXTURE_OR_TASK_NAME=\"sentiment\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"validation\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
-python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
+#python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"
 #python3 eval.py --gin_search_paths="./" --gin_file="eval_categorisation_base.gin" --gin.SPLIT=\"test\" --gin.CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000\"

finetune_categorisation_base.gin

@@ -12,8 +12,9 @@ include "t5x/configs/runs/finetune.gin"
 
 MIXTURE_OR_TASK_NAME = %gin.REQUIRED
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
-INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
-TRAIN_STEPS = 1_510_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+TRAIN_STEPS = 1_010_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
 USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
 RANDOM_SEED = 0

finetune_categorisation_large.gin

@@ -10,8 +10,8 @@ from t5x import utils
 include "t5x/examples/t5/mt5/large.gin"
 include "t5x/configs/runs/finetune.gin"
 
-MIXTURE_OR_TASK_NAME = "categorise"
-TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 2}
+MIXTURE_OR_TASK_NAME = "angry_tweets"
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 8}
 TRAIN_STEPS = 1_005_000  # 1000000 pre-trained steps + 10000 fine-tuning steps.
 USE_CACHED_TASKS = False
 DROPOUT_RATE = 0.1
@@ -25,9 +25,9 @@ RANDOM_SEED = 0
 #LOSS_NORMALIZING_FACTOR = 234496
 
 #INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
-INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000"
+#INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000"
 #INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/pk_nb_t5x_base_run1/checkpoint_1100000"
-#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_t5x_base/checkpoint_1294000"
+INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_pluss200k_scandinavian_t5x_large/checkpoint_1700000"
 
 
 

finetune_large.sh

@@ -1,7 +1,7 @@
 PROJECT_DIR=${HOME}"/models/t5-parliament-categorisation"
 T5X_DIR="../../t5x"  # directory where the t5x is cloned.
 #Needs to be updated when moving to tpu-v4  it should then be in another zone
-MODEL_DIR="gs://nb-t5x/eval_large"
+MODEL_DIR="gs://nb-t5x-us-central2/finetuned/v1_eval_angry_tweets_scandinavian_large"
 export PYTHONPATH=${PROJECT_DIR}
 
 python3 ${T5X_DIR}/t5x/train.py \

log/eval_results_t1v-n-7b23714e-w-0.jsonl

@@ -16,4 +16,6 @@
 {"model": "gs://nb-t5x-us-central2/finetuned/v1_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:01:28", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 87.29016786570743, "f1_macro": 84.21729163839953}}
 {"model": "gs://nb-t5x-us-central2/finetuned/v3_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:03:06", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 84.89208633093526, "f1_macro": 81.23942213621073}}
 {"model": "gs://nb-t5x-us-central2/finetuned/v4_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:04:20", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.61151079136691, "f1_macro": 82.17948717948718}}
-{"model": "gs://nb-t5x-us-central2/finetuned/v5_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 14:05:39", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 80.0959232613909, "f1_macro": 74.5979905029614}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "13-04-2022 20:48:48", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.27131782945736, "f1_macro": 82.54273504273503}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "14-04-2022 04:53:37", "split": "validation", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 85.27131782945736, "f1_macro": 82.54273504273503}}
+{"model": "gs://nb-t5x-us-central2/finetuned/v2_norwegian_NCC_plus_English_t5x_base_1_500_000_sentiment/checkpoint_1510000", "task": "sentiment", "eval_date": "14-04-2022 04:54:48", "split": "test", "feature_length": {"inputs": 512, "targets": 2}, "eval_batch_size": 16, "result": {"accuracy": 84.65227817745803, "f1_macro": 80.68438422789647}}

my_preprocessors.py

@@ -0,0 +1,67 @@
+import collections
+import functools
+import math
+import re
+from typing import Callable, Mapping, Optional, Sequence, Union
+import uuid
+
+from absl import logging
+import babel
+import gin
+import seqio
+import tensorflow.compat.v2 as tf
+
+import json
+import pandas as pd
+
+# We disable no-value-for-parameter since the seqio.map_over_dataset leads to
+# a false positive when seeds are provided.
+# pylint:disable=no-value-for-parameter
+AUTOTUNE = tf.data.experimental.AUTOTUNE
+
+FeatureType = Mapping[str, tf.Tensor]
+
+rekey = seqio.preprocessors.rekey
+tokenize = seqio.preprocessors.tokenize
+
+
+@seqio.map_over_dataset
+def parse_tsv(line, field_names=None, field_delim='\t'):
+  """Splits TSV lines into dict examples mapping field name to string value.
+  Args:
+    line: an example containing a comma/tab-delimited string.
+    field_names: a list of strings, the ordered names of the TSV fields.
+      Defaults to "inputs" and "targets".
+    field_delim: a string, the delimiter to split on e.g. ',' for csv.
+  Returns:
+    A feature dict mapping field name to string value.
+  """
+  field_names = field_names or ['inputs', 'targets']
+  return dict(
+      zip(field_names,
+          tf.io.decode_csv(
+              line,
+              record_defaults=[''] * len(field_names),
+              field_delim=field_delim,
+              use_quote_delim=False)))
+
+
+@seqio.map_over_dataset
+def parse_json(line,field_delim='\t'):
+  """Splits JSON lines into dict examples mapping.
+  Args:
+    line: an example containing valid json
+  Returns:
+    A feature dict mapping field name to string value.
+  """
+  mydf = pd.read_json(line, lines=True)
+  line = mydf.to_csv(header=False, index=False,sep="\t").strip()
+  field_names = list(mydf.columns)
+  
+  return dict(
+      zip(field_names,
+          tf.io.decode_csv(
+            line,
+              record_defaults=[''] * len(field_names),
+              field_delim=field_delim,
+              use_quote_delim=False)))

tasks.py

@@ -5,7 +5,8 @@ import seqio
 import my_metrics
 import tensorflow_datasets as tfds
 from t5.evaluation import metrics
-from t5.data import preprocessors
+#from t5.data import preprocessors
+import my_preprocessors
 import t5
 import tensorflow.compat.v1 as tf
 
@@ -21,6 +22,13 @@ tsv_sentiment_path = {
         "test": "gs://notram-public/finetune_datasets/norec_sentiment/test.tsv"
 }
 
+json_angry_tweets_path = {
+        "train": "gs://notram-public/finetune_datasets/angry_tweets/train.jsonl",
+        "validation": "gs://notram-public/finetune_datasets/angry_tweets/test.jsonl",
+        "test": "gs://notram-public/finetune_datasets/angry_tweets/test.jsonl"
+}
+
+
 vocabulary = seqio.SentencePieceVocabulary(
                 'gs://t5-data/vocabs/mc4.250000.100extra/sentencepiece.model', extra_ids=0)
 
@@ -52,6 +60,25 @@ def categorise_preprocessor(ds):
   return ds.map(to_inputs_and_targets, 
                 num_parallel_calls=tf.data.experimental.AUTOTUNE)
 
+def scandeval_preprocessor(ds):
+  def normalize_text(text):
+    """Lowercase and remove quotes from a TensorFlow string."""
+    text = tf.strings.regex_replace(text,"'(.*)'", r"\1")
+    return text
+
+  def to_inputs_and_targets(ex):
+    """Map {"source": ..., "source": ...}->{"target": ..., "target": ...}."""
+    return {
+        "inputs":
+             tf.strings.join(
+                 [normalize_text(ex["text"])]),
+        "targets": 
+	    tf.strings.join(
+                 [normalize_text(ex["label"])]),
+    }
+  return ds.map(to_inputs_and_targets, 
+                num_parallel_calls=tf.data.experimental.AUTOTUNE)
+
 
 seqio.TaskRegistry.add(
     "parliament",
@@ -87,3 +114,19 @@ seqio.TaskRegistry.add(
     output_features=DEFAULT_OUTPUT_FEATURES,
 )   
 
+seqio.TaskRegistry.add(
+    "angry_tweets",
+    source=seqio.TextLineDataSource(
+        split_to_filepattern=json_angry_tweets_path,
+        #num_input_examples=num_nq_examples
+        ),
+    preprocessors=[
+      functools.partial(
+          my_preprocessors.parse_tsv),
+      scandeval_preprocessor,
+      seqio.preprocessors.tokenize_and_append_eos,
+    ],
+    metric_fns=[metrics.accuracy,my_metrics.f1_macro],
+    output_features=DEFAULT_OUTPUT_FEATURES,
+)   
+