Add 1M train step model

.gitattributes

@@ -25,3 +25,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ckpt* filter=lfs diff=lfs merge=lfs -text
+*.pbtxt* filter=lfs diff=lfs merge=lfs -text

build_data.sh

@@ -0,0 +1 @@
+python3 build_pretraining_dataset.py --corpus-dir /researchdisk/training_dataset_sentences/train_splitted/ --vocab-file /researchdisk/convbert-base-finnish/vocab.txt --output-dir /researchdisk/training_dataset_sentences/train_tokenized_512 --max-seq-length 512 --num-processes 64 --no-lower-case --no-strip-accents

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/researchdisk/convbert-base-finnish",
+  "architectures": [
+    "ConvBertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "conv_kernel_size": 9,
+  "embedding_size": 768,
+  "eos_token_id": 2,
+  "head_ratio": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "convbert",
+  "num_attention_heads": 12,
+  "num_groups": 1,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.13.0.dev0",
+  "type_vocab_size": 2,
+  "vocab_size": 50265
+}

dataset_to_sentences.py

@@ -0,0 +1,23 @@
+import csv
+import datasets
+import nltk
+nltk.download('punkt')
+from nltk.tokenize import sent_tokenize
+import numpy as np
+
+
+dataset = datasets.load_from_disk("/researchdisk/training_dataset_full_deduplicated")
+
+def tokenize_sentences(example):
+    sentences = sent_tokenize(example["text"], "finnish")
+    sentences = [sentence for sentence in sentences if len(sentence.split()) >= 5]
+    sentences.append("")
+    example["text"] = "\n".join(sentences)
+    return example
+
+
+dataset["train"] = dataset["train"].map(tokenize_sentences, num_proc=64, batched=False, writer_batch_size=100000)
+dataset["validation"] = dataset["validation"].map(tokenize_sentences, num_proc=64, batched=False, writer_batch_size=100000)
+
+np.savetxt('/researchdisk/training_dataset_sentences/train.txt', dataset["train"].to_pandas().values, fmt = "%s")
+np.savetxt('/researchdisk/training_dataset_sentences/validation.txt', dataset["validation"].to_pandas().values, fmt = "%s")

events.out.tfevents.1641404286.t1v-n-8eba1090-w-0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56f18026a3d535676fb6d84a6dfbdfa5a7ae3556f144627e2fd5d1daf2b0b580
+size 48032940

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dec5473cc9fed3c62d06e25247ccffdeac12c8903f9f63d22f718f0b2977964
+size 483483789

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tf_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd2dab1e8334107ef6e3a0a35a2283d38cf26e93a8ed7ed4cbea52849bddb2e8
+size 483731808

tf_rename_checkpoint_variables.py

@@ -0,0 +1,135 @@
+# Adapted from https://gist.github.com/qqaatw/82b47c2b3da602fa1df604167bfcb9b0
+
+import getopt
+import sys
+import re
+
+import tensorflow.compat.v1 as tf
+
+
+usage_str = ('python tensorflow_rename_variables.py '
+             '--checkpoint_dir=path/to/dir/ --replace_from=substr '
+             '--replace_to=substr --add_prefix=abc --dry_run')
+find_usage_str = ('python tensorflow_rename_variables.py '
+                  '--checkpoint_dir=path/to/dir/ --find_str=[\'!\']substr')
+comp_usage_str = ('python tensorflow_rename_variables.py '
+                  '--checkpoint_dir=path/to/dir/ '
+                  '--checkpoint_dir2=path/to/dir/')
+
+
+def print_usage_str():
+    print('Please specify a checkpoint_dir. Usage:')
+    print('%s\nor\n%s\nor\n%s' % (usage_str, find_usage_str, comp_usage_str))
+    print('Note: checkpoint_dir should be a *DIR*, not a file')
+
+
+def compare(checkpoint_dir, checkpoint_dir2):
+    import difflib
+    with tf.Session():
+        list1 = [el1 for (el1, el2) in
+                 tf.train.list_variables(checkpoint_dir)]
+        list2 = [el1 for (el1, el2) in
+                 tf.train.list_variables(checkpoint_dir2)]
+        for k1 in list1:
+            if k1 in list2:
+                continue
+            else:
+                print('{} close matches: {}'.format(
+                    k1, difflib.get_close_matches(k1, list2)))
+
+
+def find(checkpoint_dir, find_str):
+    with tf.Session():
+        negate = find_str.startswith('!')
+        if negate:
+            find_str = find_str[1:]
+        for var_name, _ in tf.train.list_variables(checkpoint_dir):
+            if negate and find_str not in var_name:
+                print('%s missing from %s.' % (find_str, var_name))
+            if not negate and find_str in var_name:
+                print('Found %s in %s.' % (find_str, var_name))
+
+
+def rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run):
+    checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)
+    print('print: ', checkpoint)
+    with tf.Session() as sess:
+        for var_name, _ in tf.train.list_variables(checkpoint_dir):
+            # Load the variable
+    
+            var= tf.train.load_variable(checkpoint_dir, var_name)
+
+            # Set the new name
+            if None not in [replace_from, replace_to]:
+                new_name = re.sub(replace_from, replace_to, var_name)
+                if add_prefix:
+                    new_name = add_prefix + new_name
+                if dry_run:
+                    print('%s would be renamed to %s.' % (var_name,
+                                                            new_name))
+                else:
+                    if var_name != new_name:
+                        print('Renaming %s to %s.' % (var_name, new_name))
+                # Create the variable, potentially renaming it
+                var = tf.Variable(var, name=new_name)
+
+        if not dry_run:
+            # Save the variables
+            saver = tf.train.Saver()
+            sess.run(tf.global_variables_initializer())
+            #saver.save(sess, checkpoint.model_checkpoint_path)
+            saver.save(sess, "renamed-model.ckpt")
+
+
+def main(argv):
+    checkpoint_dir = None
+    checkpoint_dir2 = None
+    replace_from = None
+    replace_to = None
+    add_prefix = None
+    dry_run = False
+    find_str = None
+
+    try:
+        opts, args = getopt.getopt(argv, 'h', ['help=', 'checkpoint_dir=',
+                                               'replace_from=', 'replace_to=',
+                                               'add_prefix=', 'dry_run',
+                                               'find_str=',
+                                               'checkpoint_dir2='])
+    except getopt.GetoptError as e:
+        print(e)
+        print_usage_str()
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            print(usage_str)
+            sys.exit()
+        elif opt == '--checkpoint_dir':
+            checkpoint_dir = arg
+        elif opt == '--checkpoint_dir2':
+            checkpoint_dir2 = arg
+        elif opt == '--replace_from':
+            replace_from = arg
+        elif opt == '--replace_to':
+            replace_to = arg
+        elif opt == '--add_prefix':
+            add_prefix = arg
+        elif opt == '--dry_run':
+            dry_run = True
+        elif opt == '--find_str':
+            find_str = arg
+
+    if not checkpoint_dir:
+        print_usage_str()
+        sys.exit(2)
+
+    if checkpoint_dir2:
+        compare(checkpoint_dir, checkpoint_dir2)
+    elif find_str:
+        find(checkpoint_dir, find_str)
+    else:
+        rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-cased", "tokenizer_class": "BertTokenizer"}

train_tokenizer.py

@@ -0,0 +1,15 @@
+from datasets import load_from_disk
+from transformers import AutoTokenizer
+
+dataset = load_from_disk("/researchdisk/training_dataset_full_deduplicated")
+dataset = dataset["train"]
+
+# We train on batch of texts, 1000 at a time here.
+batch_size = 1000
+corpus = (dataset[i : i + batch_size]["text"] for i in range(0, len(dataset), batch_size))
+
+# ConvBERT uses Bert tokenizer
+tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+#let's use same vocab size as in Finnish-NLP/roberta-large-finnish-v2 which is also very close to TurkuNLP/bert-base-finnish-cased-v1
+new_tokenizer = tokenizer.train_new_from_iterator(corpus, vocab_size=50265)
+new_tokenizer.save_pretrained("./")